diff --git a/granularity_next_b200/tiera_moonep_probe_refresh/STATUS.txt b/granularity_next_b200/tiera_moonep_probe_refresh/STATUS.txt index 90999e07f1386b30d6b5f9f4e8feee3094db9c46..5cbde7a6f684b8f31f5abdaa428f7374f1d31519 100644 --- a/granularity_next_b200/tiera_moonep_probe_refresh/STATUS.txt +++ b/granularity_next_b200/tiera_moonep_probe_refresh/STATUS.txt @@ -1,7 +1,7 @@ study_root: /workspace/granularity_next_b200/tiera_moonep_probe_refresh gbs: 16 -updated: 2026-09-16T21:43:32+00:00 -current: idle +updated: 2026-09-21T05:22:00+00:00 +current: starting completed steps: moonep__tierbprobe__gbs16 diff --git a/granularity_next_b200/tiera_moonep_probe_refresh/collected_all.tsv b/granularity_next_b200/tiera_moonep_probe_refresh/collected_all.tsv index 81f9c93c168fd6bf7ad44fb5cf97569b76f03ddc..276ffee5a9c5f5aa5d3a7a4aef9cac01a58ed250 100644 --- a/granularity_next_b200/tiera_moonep_probe_refresh/collected_all.tsv +++ b/granularity_next_b200/tiera_moonep_probe_refresh/collected_all.tsv @@ -1,9 +1,9 @@ run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning -moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 tierbprobe 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 OK 12 597.53 584.50 18.18 558.4 92581 59020 -moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 OK 12 573.83 569.10 4.07 581.0 92581 59020 -moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 tierbprobe 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 OK 12 650.03 566.50 137.54 529.7 95485 56290 -moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 OK 12 551.20 545.40 4.35 604.8 95485 56290 -moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 tierbprobe 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 OK 12 611.76 607.20 5.12 545.0 98677 55690 -moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 OK 12 708.73 601.50 184.17 492.1 98613 55624 -moonep/tierbprobe/tierbprobe_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 tierbprobe 0 1 G4 moonep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 OK 12 791.24 788.30 3.38 421.3 105107 57126 -moonep/tierbprobe/tierbprobe_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 G4 moonep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 OK 12 787.61 783.30 3.15 423.3 105107 57126 +moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 tierbprobe 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 INCOMPLETE 0 +moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 INCOMPLETE 0 +moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 tierbprobe 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 INCOMPLETE 0 +moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 NOLOG 0 +moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 tierbprobe 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 INCOMPLETE 0 +moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 NOLOG 0 +moonep/tierbprobe/tierbprobe_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 tierbprobe 0 1 G4 moonep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 NOLOG 0 +moonep/tierbprobe/tierbprobe_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 G4 moonep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 0 16 aux_loss 1e-2 0 0 1000000 0 512 128 64 128 1.0 softmax 0 0 0 0 0 0.1 32768 0.01 NOLOG 0 diff --git a/granularity_next_b200/tiera_moonep_probe_refresh/hf_watcher.log b/granularity_next_b200/tiera_moonep_probe_refresh/hf_watcher.log index 169b9d384a01a5a37c136f5f5cfee599253a28f2..cb3dfcc1ce874d0753fd05a485e68500230cd7e9 100644 --- a/granularity_next_b200/tiera_moonep_probe_refresh/hf_watcher.log +++ b/granularity_next_b200/tiera_moonep_probe_refresh/hf_watcher.log @@ -6,3 +6,4 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main [2026-09-16T21:43:29+00:00] backup pass uploaded: /workspace/granularity_next_b200/tiera_moonep_probe_refresh -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tiera_moonep_probe_refresh https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +Already logged in to Hugging Face as: AbhishekGhosh1998 diff --git a/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log b/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log index 206e58098bf6a22b3e9df745afbaed1302a8bbe2..d6a5c8999aa4c19562f2328cbc279855e1805094 100644 --- a/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log +++ b/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log @@ -1,4838 +1,3 @@ -[21:33:42] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -============================================================== - G1 / moonep (slots/rank=1 policy=matched) - E=8 K=2 H'=14336 m=1 - L=16 seq=4096 gbs=16 (2 microbatches) - cap_mode=none cap=none cg=0 recompute=none - memory_dump=none snapshots=/workspace/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ a6a1a221 -============================================================== -W0916 21:33:51.335000 28497 torch/distributed/run.py:766] -W0916 21:33:51.335000 28497 torch/distributed/run.py:766] ***************************************** -W0916 21:33:51.335000 28497 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0916 21:33:51.335000 28497 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.auto - attention_dropout ............................... 0.0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_scope ................................ full - cuda_graph_warmup_steps ......................... 3 - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 8 - data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_type ................................. single - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - delay_wgrad_compute ............................. False - deprecated_use_mcore_models ..................... True - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_bf16_reduced_precision_matmul ........... False - disable_mamba_mem_eff_path ...................... False - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - embedding_init_method_std ....................... None - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_experimental ............................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_msc ...................................... True - enable_one_logger ............................... True - encoder_num_layers .............................. 16 - encoder_seq_length .............................. 4096 - end_weight_decay ................................ 0.1 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000000 - eval_iters ...................................... 0 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 8 - expert_tensor_parallel_size ..................... 1 - export_force_local_attention .................... False - export_kd_cfg ................................... None - export_kd_teacher_ckpt_format ................... None - export_kd_teacher_load .......................... None - export_kv_cache_quant ........................... False - export_legacy_megatron .......................... False - export_model_type ............................... GPTModel - export_moe_apply_probs_on_input ................. False - export_qk_l2_norm ............................... False - export_quant_cfg ................................ None - export_real_quant_cfg ........................... None - export_te_mcore_model ........................... False - external_cuda_graph ............................. False - ffn_hidden_size ................................. 14336 - finetune ........................................ False - finetune_data_split ............................. train - finetune_hf_dataset ............................. None - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. e4m3 - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... mxfp8 - fp8_wgrad ....................................... True - fsdp_double_buffer .............................. False - global_batch_size ............................... 16 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... True - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - heterogeneous_layers_config_encoded_json ........ None - heterogeneous_layers_config_path ................ None - hidden_dropout .................................. 0.0 - hidden_size ..................................... 4096 - hierarchical_context_parallel_sizes ............. None - high_priority_stream_groups ..................... [] - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_dynamic_batching ...................... False - inference_dynamic_batching_buffer_guaranteed_fraction 0.2 - inference_dynamic_batching_buffer_overflow_factor None - inference_dynamic_batching_buffer_size_gb ....... 40.0 - inference_dynamic_batching_chunk_size ........... 256 - inference_dynamic_batching_max_requests_override None - inference_dynamic_batching_max_tokens_override .. None - inference_dynamic_batching_num_cuda_graphs ...... 16 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.01 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 4294967296 - inprocess_active_world_size ..................... 8 - inprocess_barrier_timeout ....................... 120 - inprocess_completion_timeout .................... 120 - inprocess_empty_cuda_cache ...................... False - inprocess_granularity ........................... node - inprocess_hard_timeout .......................... 90 - inprocess_heartbeat_interval .................... 30 - inprocess_heartbeat_timeout ..................... 60 - inprocess_last_call_wait ........................ 1 - inprocess_max_iterations ........................ None - inprocess_monitor_process_interval .............. 1.0 - inprocess_monitor_thread_interval ............... 1.0 - inprocess_progress_watchdog_interval ............ 1.0 - inprocess_restart ............................... False - inprocess_soft_timeout .......................... 60 - inprocess_termination_grace_time ................ 1 - is_hybrid_model ................................. False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kitchen_config_file ............................. None - kitchen_recipe_number ........................... None - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - lazy_mpu_init ................................... None - load ............................................ None - load_main_params_from_ckpt ...................... None - load_model_opt_format ........................... False - local_rank ...................................... 0 - log_energy ...................................... False - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. True - log_timers_to_tensorboard ....................... False - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 8 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. None - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 5 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - mamba_head_dim .................................. 64 - mamba_num_groups ................................ 8 - mamba_num_heads ................................. None - mamba_state_dim ................................. 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... False - max_position_embeddings ......................... 32768 - max_tokens_to_oom ............................... 12000 - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - mid_level_dataset_surplus ....................... 0.005 - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-05 - mlp_chunks_for_prefill .......................... 1 - mmap_bin_files .................................. True - mock_data ....................................... False - moe_apply_probs_on_input ........................ False - moe_aux_loss_coeff .............................. 0.01 - moe_deepep_num_sms .............................. 20 - moe_echo_enable_random_offloading ............... False - moe_echo_expert_dispatch_overlap ................ False - moe_echo_expert_dispatcher_type ................. hybridep - moe_echo_recompute_expert_dispatch .............. False - moe_enable_deepep ............................... False - moe_enable_echo ................................. False - moe_enable_hybridep ............................. False - moe_enable_moonep ............................... True - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. 14336 - moe_grouped_gemm ................................ True - moe_hybridep_num_sms ............................ 16 - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_moonep_async_grad_reduce .................... True - moe_moonep_num_sms .............................. None - moe_moonep_traffic_logging ...................... True - moe_moonep_traffic_logging_interval ............. 1 - moe_moonep_traffic_warmup_iters ................. 2 - moe_num_echo_experts ............................ None - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_received_token_capacity ..................... None - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ fp32 - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... flex - moe_token_drop_policy ........................... probs - moe_upcycling_granularity ....................... 1 - moe_use_device_initiated_grouped_gemm ........... True - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mrope_section ................................... None - mscale .......................................... 1.0 - mscale_all_dim .................................. 0.0 - mtp_loss_scaling_factor ......................... 0.1 - mtp_num_layers .................................. None - multi_latent_attention .......................... False - nccl_all_reduce_for_prefill ..................... False - nccl_communicator_config_path ................... None - nccl_ub ......................................... False - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_rope_freq .................................... None - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-05 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... 8 - num_layers ...................................... 16 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 2 - object_storage_cache_path ....................... None - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_grad_reduce ............................. True - overlap_moe_expert_parallel_comm ................ False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ True - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_layout .................. None - pipeline_model_parallel_size .................... 1 - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_l2_norm ...................................... False - qk_layernorm .................................... False - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... None - recompute_method ................................ None - recompute_modules ............................... None - recompute_num_layers ............................ None - record_memory_history ........................... False - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... validate_results - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - reuse_grad_buf_for_mxfp8_param_ag ............... False - rope_scaling_factor ............................. 8.0 - rope_type ....................................... None - rotary_base ..................................... 1000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - run_workload_inspector_server ................... False - sample_rate ..................................... 1.0 - save ............................................ None - save_interval ................................... None - save_retain_interval ............................ None - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - seq_length ...................................... 4096 - sequence_parallel ............................... False - sft ............................................. False - sft_tokenizer_prompt_format ..................... nemotron-h-aligned - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - squared_relu .................................... False - start_weight_decay .............................. 0.1 - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - suggested_communication_unit_size ............... None - swiglu .......................................... True - swin_backbone_type .............................. tiny - symmetric_ar_type ............................... None - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. None - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. GPT2BPETokenizer - torch_fsdp2_reshard_after_forward ............... True - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 12 - train_samples ................................... None - train_sync_interval ............................. None - transformer_impl ................................ transformer_engine - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... True - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_sharp ....................................... False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ False - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json - vocab_size ...................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.1 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 8 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> building GPT2BPETokenizer tokenizer ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") - > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS -> initializing torch distributed ... -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 -7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 3 is connected to 71 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : -7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank [Gloo] Rank 00[Gloo] Rank is connected to 0 is connected to 0 is connected to 0 peer ranks. 0 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : 0 -0 - -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank5]:[W916 21:34:07.419436954 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W916 21:34:07.419455400 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W916 21:34:07.436207255 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W916 21:34:07.440853164 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W916 21:34:07.440888946 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W916 21:34:07.440913588 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W916 21:34:07.440920898 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -> compiling dataset index builder ... -make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.034 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -[rank0]:[W916 21:34:07.611856603 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.892 seconds -time to initialize megatron (seconds): 8.289 -[after megatron is initialized] datetime: 2026-09-16 21:34:14 -building GPT model ... -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3902410752 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 -Params for bucket 1 (206045184 elements, 206045184 padded size): - module.output_layer.weight -Params for bucket 2 (41988096 elements, 41988096 padded size): - module.decoder.final_layernorm.weight - module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.self_attention.linear_qkv.weight - module.decoder.layers.15.mlp.router.weight - module.decoder.layers.15.self_attention.linear_proj.weight -Params for bucket 3 (41984000 elements, 41984000 padded size): - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.14.self_attention.linear_qkv.weight - module.decoder.layers.14.pre_mlp_layernorm.weight -Params for bucket 4 (41984000 elements, 41984000 padded size): - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.13.self_attention.linear_qkv.weight - module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.13.pre_mlp_layernorm.weight -Params for bucket 5 (41984000 elements, 41984000 padded size): - module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.12.self_attention.linear_qkv.weight -Params for bucket 6 (41984000 elements, 41984000 padded size): - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.self_attention.linear_qkv.weight - module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.11.pre_mlp_layernorm.weight -Params for bucket 7 (41984000 elements, 41984000 padded size): - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.10.pre_mlp_layernorm.weight -Params for bucket 8 (41984000 elements, 41984000 padded size): - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_qkv.weight - module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight -Params for bucket 9 (41984000 elements, 41984000 padded size): - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.8.pre_mlp_layernorm.weight -Params for bucket 10 (41984000 elements, 41984000 padded size): - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.7.pre_mlp_layernorm.weight -Params for bucket 11 (41984000 elements, 41984000 padded size): - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.linear_proj.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight -Params for bucket 12 (41984000 elements, 41984000 padded size): - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight -Params for bucket 13 (41984000 elements, 41984000 padded size): - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.self_attention.linear_qkv.weight -Params for bucket 14 (41984000 elements, 41984000 padded size): - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.3.self_attention.linear_qkv.weight -Params for bucket 15 (41984000 elements, 41984000 padded size): - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.2.pre_mlp_layernorm.weight -Params for bucket 16 (41984000 elements, 41984000 padded size): - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.1.pre_mlp_layernorm.weight -Params for bucket 17 (41984000 elements, 41984000 padded size): - module.decoder.layers.0.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.linear_qkv.weight -Params for bucket 18 (206045184 elements, 206045184 padded size): - module.embedding.word_embeddings.weight -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 32 -Params for bucket 1 (58720256 elements, 58720256 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight0 -Params for bucket 2 (117440512 elements, 117440512 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 -Params for bucket 3 (58720256 elements, 58720256 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight0 -Params for bucket 4 (117440512 elements, 117440512 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 -Params for bucket 5 (58720256 elements, 58720256 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 -Params for bucket 6 (117440512 elements, 117440512 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight0 -Params for bucket 7 (58720256 elements, 58720256 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight0 -Params for bucket 8 (117440512 elements, 117440512 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight0 -Params for bucket 9 (58720256 elements, 58720256 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 -Params for bucket 10 (117440512 elements, 117440512 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 -Params for bucket 11 (58720256 elements, 58720256 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 -Params for bucket 12 (117440512 elements, 117440512 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 -Params for bucket 13 (58720256 elements, 58720256 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 -Params for bucket 14 (117440512 elements, 117440512 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 -Params for bucket 15 (58720256 elements, 58720256 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 -Params for bucket 16 (117440512 elements, 117440512 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 -Params for bucket 17 (58720256 elements, 58720256 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 -Params for bucket 18 (117440512 elements, 117440512 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 -Params for bucket 19 (58720256 elements, 58720256 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 -Params for bucket 20 (117440512 elements, 117440512 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 -Params for bucket 21 (58720256 elements, 58720256 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight0 -Params for bucket 22 (117440512 elements, 117440512 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 -Params for bucket 23 (58720256 elements, 58720256 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 -Params for bucket 24 (117440512 elements, 117440512 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 -Params for bucket 25 (58720256 elements, 58720256 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 -Params for bucket 26 (117440512 elements, 117440512 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 -Params for bucket 27 (58720256 elements, 58720256 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 -Params for bucket 28 (117440512 elements, 117440512 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 -Params for bucket 29 (58720256 elements, 58720256 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 -Params for bucket 30 (117440512 elements, 117440512 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 -Params for bucket 31 (58720256 elements, 58720256 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 -Params for bucket 32 (117440512 elements, 117440512 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-16 21:34:17 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 192 - validation: 0 - test: 0 -INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] -> building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) -INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers -INFO:megatron.core.datasets.indexed_dataset: Extract the document indices -INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 -INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 -INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices -INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 -INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 -> finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-09-16 21:34:17 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (2817.52, 2819.60) - train/valid/test-data-iterators-setup ..........: (128.43, 135.28) -training ... -Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-09-16 21:34:17 -INFO:moonep.api:MoonEP comm buffer sizes: - config : R=8, S=4096, H=4096, E=8, K=2, NvS=8446, NvS_padded=8448 - sm_count : num_sms=32, num_sms_dedup=148 - hidden_buffer_size : 66.00 MiB - meta_buffer_size : 2.00 MiB - local_temp_buffer_size : 512.66 KiB - total_size : 68.50 MiB -[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 8] sizes [768, 7552] | duplicated experts [[1], [-1], [4], [1], [1], [1], [4], [4]] -[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 8] sizes [7808, 512] | duplicated experts [[4], [6], [4], [-1], [3], [7], [0], [4]] -[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 8] sizes [8064, 256] | duplicated experts [[3], [3], [-1], [2], [0], [2], [2], [5]] -[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 8] sizes [8192, 0] | duplicated experts [[-1], [0], [1], [1], [0], [1], [1], [1]] -WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-09-16 21:34:39] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 21771.7 | throughput per GPU (TFLOP/s/GPU): 15.3 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.104376E+01 | loss scale: 1.0 | grad norm: 35.707 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 23.22 -Number of parameters in embedding layers in billions: 0.41 -Total number of parameters in billions: 23.63 -Number of parameters in most loaded shard in billions: 23.6320 -Theoretical memory footprints: weight and optimizer=169029.43 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 53315.5478515625 | max allocated: 53315.548828125 | reserved: 59020.0 | max reserved: 59020.0 - [2026-09-16 21:34:39] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 621.1 | throughput per GPU (TFLOP/s/GPU): 536.8 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.104935E+01 | loss scale: 1.0 | grad norm: 37.624 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31846 payload_rows_remote=24080 dup_rows=922 dup_rows_remote=842 dedup_ratio=0.028137 router_tokens_per_expert=[186, 1345, 2668, 995, 2434, 1491, 3528, 3737] payload_rows_per_dst=[7766, 7768, 5934, 7396, 0, 2982, 0, 0] dup_rows_per_dst=[80, 404, 146, 292, 0, 0, 0, 0] combine_rows_per_src=[7766, 7768, 5934, 7396, 0, 2982, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32628 payload_rows_remote=29276 dup_rows=140 dup_rows_remote=84 dedup_ratio=0.004272 router_tokens_per_expert=[245, 873, 1694, 684, 4334, 1510, 3158, 3886] payload_rows_per_dst=[7614, 1746, 3388, 3352, 7498, 3020, 5446, 564] dup_rows_per_dst=[84, 0, 0, 56, 0, 0, 0, 0] combine_rows_per_src=[7614, 1746, 3388, 3352, 7498, 3020, 5446, 564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32494 payload_rows_remote=25528 dup_rows=274 dup_rows_remote=274 dedup_ratio=0.008362 router_tokens_per_expert=[110, 1152, 2786, 891, 2790, 1273, 3483, 3899] payload_rows_per_dst=[220, 2304, 1964, 1782, 5756, 5704, 6966, 7798] dup_rows_per_dst=[0, 0, 0, 0, 44, 230, 0, 0] combine_rows_per_src=[220, 2304, 1964, 1782, 5756, 5704, 6966, 7798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=29346 dup_rows=34 dup_rows_remote=0 dedup_ratio=0.001038 router_tokens_per_expert=[72, 935, 2781, 528, 2835, 1511, 3852, 3870] payload_rows_per_dst=[144, 1870, 5162, 1056, 5670, 3388, 7704, 7740] dup_rows_per_dst=[0, 0, 0, 0, 0, 34, 0, 0] combine_rows_per_src=[144, 1870, 5162, 1056, 5670, 3388, 7704, 7740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32112 payload_rows_remote=23008 dup_rows=656 dup_rows_remote=570 dedup_ratio=0.020020 router_tokens_per_expert=[177, 774, 1693, 889, 4286, 1491, 2711, 4363] payload_rows_per_dst=[354, 1548, 0, 1778, 6022, 8348, 4958, 9104] dup_rows_per_dst=[0, 0, 0, 0, 0, 570, 0, 86] combine_rows_per_src=[354, 1548, 0, 1778, 6022, 8348, 4958, 9104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31712 payload_rows_remote=25112 dup_rows=1056 dup_rows_remote=700 dedup_ratio=0.032227 router_tokens_per_expert=[227, 1532, 2822, 871, 2773, 1317, 3031, 3811] payload_rows_per_dst=[7994, 5872, 6600, 7138, 1474, 2634, 0, 0] dup_rows_per_dst=[82, 384, 356, 234, 0, 0, 0, 0] combine_rows_per_src=[7994, 5872, 6600, 7138, 1474, 2634, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31760 payload_rows_remote=22838 dup_rows=1008 dup_rows_remote=340 dedup_ratio=0.030762 router_tokens_per_expert=[174, 1290, 2224, 1013, 2710, 1193, 3853, 3927] payload_rows_per_dst=[8138, 8922, 4448, 7866, 0, 2386, 0, 0] dup_rows_per_dst=[64, 668, 0, 276, 0, 0, 0, 0] combine_rows_per_src=[8138, 8922, 4448, 7866, 0, 2386, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=197263360 dispatch_meta_bytes=99688 combine_payload_bytes=197263360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=197363048 ingress_bytes=1606549504 link_bytes=1606549504 token_bytes=394626408 weight_movement_bytes=1409286144 total_nvlink_bytes=1803912552 weight_to_token_ratio=3.571191 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.996006616e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=209125376 dispatch_meta_bytes=103208 combine_payload_bytes=209125376 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=209228584 ingress_bytes=5141626880 link_bytes=5141626880 token_bytes=418353960 weight_movement_bytes=4932501504 total_nvlink_bytes=5350855464 weight_to_token_ratio=11.790259 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.879096431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=239828992 dispatch_meta_bytes=117440 combine_payload_bytes=239828992 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=239946432 ingress_bytes=1649115136 link_bytes=1649115136 token_bytes=479775424 weight_movement_bytes=1409286144 total_nvlink_bytes=1889061568 weight_to_token_ratio=2.937387 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3840 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.234356387e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=240402432 dispatch_meta_bytes=117384 combine_payload_bytes=240402432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=240519816 ingress_bytes=1649688576 link_bytes=1649688576 token_bytes=480922248 weight_movement_bytes=1409286144 total_nvlink_bytes=1890208392 weight_to_token_ratio=2.930383 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6016 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.237567412e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26464 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[114, 641, 2385, 771, 3152, 1736, 3847, 3738] payload_rows_per_dst=[228, 1282, 4770, 1542, 6304, 3472, 7694, 7476] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[228, 1282, 4770, 1542, 6304, 3472, 7694, 7476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=188481536 dispatch_meta_bytes=94312 combine_payload_bytes=188481536 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=188575848 ingress_bytes=3007053824 link_bytes=3007053824 token_bytes=377057384 weight_movement_bytes=2818572288 total_nvlink_bytes=3195629672 weight_to_token_ratio=7.475181 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.683824621e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=205717504 dispatch_meta_bytes=103248 combine_payload_bytes=205717504 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=205820752 ingress_bytes=3024289792 link_bytes=3024289792 token_bytes=411538256 weight_movement_bytes=2818572288 total_nvlink_bytes=3230110544 weight_to_token_ratio=6.848871 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.693476043e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=187088896 dispatch_meta_bytes=92712 combine_payload_bytes=187088896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=187181608 ingress_bytes=1596375040 link_bytes=1596375040 token_bytes=374270504 weight_movement_bytes=1409286144 total_nvlink_bytes=1783556648 weight_to_token_ratio=3.765421 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4864 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.939033864e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31702 payload_rows_remote=22882 dup_rows=1066 dup_rows_remote=1066 dedup_ratio=0.032532 router_tokens_per_expert=[4410, 2284, 642, 7331, 480, 383, 220, 634] payload_rows_per_dst=[8820, 8916, 8724, 1808, 960, 766, 440, 1268] dup_rows_per_dst=[0, 888, 178, 0, 0, 0, 0, 0] combine_rows_per_src=[8820, 8916, 8724, 1808, 960, 766, 440, 1268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=216793088 dispatch_meta_bytes=105856 combine_payload_bytes=216793088 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=216898944 ingress_bytes=3740008448 link_bytes=3740008448 token_bytes=433692032 weight_movement_bytes=3523215360 total_nvlink_bytes=3956907392 weight_to_token_ratio=8.123772 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.094248615e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31710 payload_rows_remote=20200 dup_rows=1058 dup_rows_remote=586 dedup_ratio=0.032288 router_tokens_per_expert=[3017, 3728, 289, 7760, 489, 530, 37, 534] payload_rows_per_dst=[0, 0, 3348, 0, 7670, 11510, 722, 8460] dup_rows_per_dst=[0, 0, 34, 0, 486, 472, 2, 64] combine_rows_per_src=[0, 0, 3348, 0, 7670, 11510, 722, 8460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32432 payload_rows_remote=17942 dup_rows=336 dup_rows_remote=282 dedup_ratio=0.010254 router_tokens_per_expert=[4559, 2799, 269, 7438, 335, 331, 56, 597] payload_rows_per_dst=[0, 0, 4120, 0, 670, 6468, 14490, 6684] dup_rows_per_dst=[0, 0, 2, 0, 0, 172, 54, 108] combine_rows_per_src=[0, 0, 4120, 0, 670, 6468, 14490, 6684] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31950 payload_rows_remote=24354 dup_rows=818 dup_rows_remote=818 dedup_ratio=0.024963 router_tokens_per_expert=[2928, 2774, 563, 7806, 688, 650, 341, 634] payload_rows_per_dst=[5856, 4370, 1126, 7596, 8582, 1300, 682, 2438] dup_rows_per_dst=[0, 0, 0, 0, 810, 0, 0, 8] combine_rows_per_src=[5856, 4370, 1126, 7596, 8582, 1300, 682, 2438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=187449344 dispatch_meta_bytes=95792 combine_payload_bytes=187449344 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=187545136 ingress_bytes=3710664704 link_bytes=3710664704 token_bytes=374994480 weight_movement_bytes=3523215360 total_nvlink_bytes=3898209840 weight_to_token_ratio=9.395379 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.077817343e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32312 payload_rows_remote=25400 dup_rows=456 dup_rows_remote=364 dedup_ratio=0.013916 router_tokens_per_expert=[4039, 3075, 420, 7620, 438, 214, 151, 427] payload_rows_per_dst=[0, 0, 3406, 1234, 876, 4628, 15256, 6912] dup_rows_per_dst=[0, 0, 46, 0, 0, 32, 286, 92] combine_rows_per_src=[0, 0, 3406, 1234, 876, 4628, 15256, 6912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32686 payload_rows_remote=32346 dup_rows=82 dup_rows_remote=82 dedup_ratio=0.002502 router_tokens_per_expert=[4587, 2859, 170, 7247, 484, 248, 44, 745] payload_rows_per_dst=[9174, 5718, 340, 13210, 2170, 496, 88, 1490] dup_rows_per_dst=[0, 0, 0, 0, 82, 0, 0, 0] combine_rows_per_src=[9174, 5718, 340, 13210, 2170, 496, 88, 1490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=165478400 dispatch_meta_bytes=83144 combine_payload_bytes=165478400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=165561544 ingress_bytes=1574764544 link_bytes=1574764544 token_bytes=331039944 weight_movement_bytes=1409286144 total_nvlink_bytes=1740326088 weight_to_token_ratio=4.257148 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1792 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.818024107e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=146980864 dispatch_meta_bytes=72896 combine_payload_bytes=146980864 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=147053760 ingress_bytes=1556267008 link_bytes=1556267008 token_bytes=294034624 weight_movement_bytes=1409286144 total_nvlink_bytes=1703320768 weight_to_token_ratio=4.792926 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.714445627e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=199507968 dispatch_meta_bytes=100688 combine_payload_bytes=199507968 prefetch_bytes=1409286144 reduce_bytes=4932501504 egress_bytes=199608656 ingress_bytes=6541295616 link_bytes=6541295616 token_bytes=399116624 weight_movement_bytes=6341787648 total_nvlink_bytes=6740904272 weight_to_token_ratio=15.889560 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.662852498e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=208076800 dispatch_meta_bytes=103056 combine_payload_bytes=208076800 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=208179856 ingress_bytes=1617362944 link_bytes=1617362944 token_bytes=416256656 weight_movement_bytes=1409286144 total_nvlink_bytes=1825542800 weight_to_token_ratio=3.385618 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2432 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.056557366e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=264978432 dispatch_meta_bytes=129712 combine_payload_bytes=264978432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=265108144 ingress_bytes=1674264576 link_bytes=1674264576 token_bytes=530086576 weight_movement_bytes=1409286144 total_nvlink_bytes=1939372720 weight_to_token_ratio=2.658596 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.375182753e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31458 payload_rows_remote=23938 dup_rows=1310 dup_rows_remote=1284 dedup_ratio=0.039978 router_tokens_per_expert=[606, 477, 811, 293, 3167, 185, 4320, 6525] payload_rows_per_dst=[7520, 14092, 6300, 3176, 0, 370, 0, 0] dup_rows_per_dst=[26, 770, 484, 30, 0, 0, 0, 0] combine_rows_per_src=[7520, 14092, 6300, 3176, 0, 370, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31926 payload_rows_remote=22772 dup_rows=842 dup_rows_remote=424 dedup_ratio=0.025696 router_tokens_per_expert=[4790, 2182, 281, 7381, 474, 710, 157, 409] payload_rows_per_dst=[2942, 4238, 3862, 1556, 9154, 6022, 314, 3838] dup_rows_per_dst=[0, 0, 2, 0, 418, 350, 0, 72] combine_rows_per_src=[2942, 4238, 3862, 1556, 9154, 6022, 314, 3838] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=24064 dup_rows=66 dup_rows_remote=66 dedup_ratio=0.002014 router_tokens_per_expert=[2988, 4319, 545, 6927, 445, 276, 217, 667] payload_rows_per_dst=[5976, 8638, 7514, 7364, 890, 552, 434, 1334] dup_rows_per_dst=[0, 0, 66, 0, 0, 0, 0, 0] combine_rows_per_src=[5976, 8638, 7514, 7364, 890, 552, 434, 1334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30660 payload_rows_remote=23980 dup_rows=2108 dup_rows_remote=2022 dedup_ratio=0.064331 router_tokens_per_expert=[236, 518, 113, 830, 4656, 131, 2726, 7174] payload_rows_per_dst=[472, 1036, 226, 1660, 9312, 6680, 10692, 582] dup_rows_per_dst=[0, 0, 0, 0, 0, 86, 2022, 0] combine_rows_per_src=[472, 1036, 226, 1660, 9312, 6680, 10692, 582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32602 payload_rows_remote=24938 dup_rows=166 dup_rows_remote=0 dedup_ratio=0.005066 router_tokens_per_expert=[267, 930, 129, 1194, 2431, 133, 3637, 7663] payload_rows_per_dst=[534, 1860, 258, 2388, 4862, 266, 7664, 14770] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 166, 0] combine_rows_per_src=[534, 1860, 258, 2388, 4862, 266, 7664, 14770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=196100096 dispatch_meta_bytes=100888 combine_payload_bytes=196100096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=196200984 ingress_bytes=1605386240 link_bytes=1605386240 token_bytes=392301080 weight_movement_bytes=1409286144 total_nvlink_bytes=1801587224 weight_to_token_ratio=3.592359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2304 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.989492823e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=186548224 dispatch_meta_bytes=92784 combine_payload_bytes=186548224 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=186641008 ingress_bytes=1595834368 link_bytes=1595834368 token_bytes=373189232 weight_movement_bytes=1409286144 total_nvlink_bytes=1782475376 weight_to_token_ratio=3.776331 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.936006327e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=197132288 dispatch_meta_bytes=96520 combine_payload_bytes=197132288 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=197228808 ingress_bytes=3015704576 link_bytes=3015704576 token_bytes=394361096 weight_movement_bytes=2818572288 total_nvlink_bytes=3212933384 weight_to_token_ratio=7.147186 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6784 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.688668681e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31250 payload_rows_remote=15352 dup_rows=1518 dup_rows_remote=0 dedup_ratio=0.046326 router_tokens_per_expert=[373, 144, 62, 475, 2460, 124, 4831, 7915] payload_rows_per_dst=[746, 288, 124, 950, 3334, 248, 9662, 15898] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 1518] combine_rows_per_src=[746, 288, 124, 950, 3334, 248, 9662, 15898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=196444160 dispatch_meta_bytes=104008 combine_payload_bytes=196444160 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=196548168 ingress_bytes=1605730304 link_bytes=1605730304 token_bytes=392992328 weight_movement_bytes=1409286144 total_nvlink_bytes=1802278472 weight_to_token_ratio=3.586040 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.991419438e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32364 payload_rows_remote=23000 dup_rows=404 dup_rows_remote=328 dedup_ratio=0.012329 router_tokens_per_expert=[379, 386, 219, 452, 2615, 341, 4332, 7660] payload_rows_per_dst=[2556, 1730, 3970, 9364, 8370, 6374, 0, 0] dup_rows_per_dst=[102, 14, 126, 76, 0, 86, 0, 0] combine_rows_per_src=[2556, 1730, 3970, 9364, 8370, 6374, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=204292096 dispatch_meta_bytes=99752 combine_payload_bytes=204292096 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=204391848 ingress_bytes=3727507456 link_bytes=3727507456 token_bytes=408683944 weight_movement_bytes=3523215360 total_nvlink_bytes=3931899304 weight_to_token_ratio=8.620880 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7040 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.087248582e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32360 payload_rows_remote=23456 dup_rows=408 dup_rows_remote=246 dedup_ratio=0.012451 router_tokens_per_expert=[778, 318, 266, 173, 4401, 1047, 1687, 7714] payload_rows_per_dst=[10226, 2212, 8904, 8924, 0, 2094, 0, 0] dup_rows_per_dst=[132, 0, 162, 114, 0, 0, 0, 0] combine_rows_per_src=[10226, 2212, 8904, 8924, 0, 2094, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=125763584 dispatch_meta_bytes=61408 combine_payload_bytes=125763584 prefetch_bytes=704643072 reduce_bytes=5637144576 egress_bytes=125824992 ingress_bytes=6467551232 link_bytes=6467551232 token_bytes=251588576 weight_movement_bytes=6341787648 total_nvlink_bytes=6593376224 weight_to_token_ratio=25.206978 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.621558721e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=188416000 dispatch_meta_bytes=93312 combine_payload_bytes=188416000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=188509312 ingress_bytes=1597702144 link_bytes=1597702144 token_bytes=376925312 weight_movement_bytes=1409286144 total_nvlink_bytes=1786211456 weight_to_token_ratio=3.738900 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2944 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.946465093e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32084 payload_rows_remote=27040 dup_rows=684 dup_rows_remote=160 dedup_ratio=0.020874 router_tokens_per_expert=[1534, 267, 3091, 45, 1309, 3798, 6269, 71] payload_rows_per_dst=[5044, 12912, 6182, 90, 2618, 5096, 0, 142] dup_rows_per_dst=[524, 160, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5044, 12912, 6182, 90, 2618, 5096, 0, 142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=192151552 dispatch_meta_bytes=94808 combine_payload_bytes=192151552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=192246360 ingress_bytes=1601437696 link_bytes=1601437696 token_bytes=384397912 weight_movement_bytes=1409286144 total_nvlink_bytes=1793684056 weight_to_token_ratio=3.666217 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1408 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.967382625e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32420 payload_rows_remote=30606 dup_rows=348 dup_rows_remote=348 dedup_ratio=0.010620 router_tokens_per_expert=[2340, 114, 4430, 5, 146, 2470, 6451, 428] payload_rows_per_dst=[4680, 228, 3292, 10, 11076, 1814, 1924, 9396] dup_rows_per_dst=[0, 0, 0, 0, 194, 0, 0, 154] combine_rows_per_src=[4680, 228, 3292, 10, 11076, 1814, 1924, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32392 payload_rows_remote=25502 dup_rows=376 dup_rows_remote=376 dedup_ratio=0.011475 router_tokens_per_expert=[372, 1377, 108, 311, 4573, 276, 1784, 7583] payload_rows_per_dst=[3314, 2754, 216, 1296, 6890, 15360, 2562, 0] dup_rows_per_dst=[16, 0, 0, 2, 0, 358, 0, 0] combine_rows_per_src=[3314, 2754, 216, 1296, 6890, 15360, 2562, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32168 payload_rows_remote=24538 dup_rows=600 dup_rows_remote=218 dedup_ratio=0.018311 router_tokens_per_expert=[546, 575, 51, 972, 3016, 423, 3216, 7585] payload_rows_per_dst=[7010, 7630, 11940, 4742, 0, 846, 0, 0] dup_rows_per_dst=[114, 382, 58, 46, 0, 0, 0, 0] combine_rows_per_src=[7010, 7630, 11940, 4742, 0, 846, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32658 payload_rows_remote=18420 dup_rows=110 dup_rows_remote=110 dedup_ratio=0.003357 router_tokens_per_expert=[1956, 15, 2096, 237, 510, 3864, 7119, 587] payload_rows_per_dst=[3912, 30, 2356, 474, 1020, 3024, 14238, 7604] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 110] combine_rows_per_src=[3912, 30, 2356, 474, 1020, 3024, 14238, 7604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=221511680 dispatch_meta_bytes=108800 combine_payload_bytes=221511680 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=221620480 ingress_bytes=1630797824 link_bytes=1630797824 token_bytes=443132160 weight_movement_bytes=1409286144 total_nvlink_bytes=1852418304 weight_to_token_ratio=3.180284 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7808 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.131787086e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=250724352 dispatch_meta_bytes=123816 combine_payload_bytes=250724352 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=250848168 ingress_bytes=4478582784 link_bytes=4478582784 token_bytes=501572520 weight_movement_bytes=4227858432 total_nvlink_bytes=4729430952 weight_to_token_ratio=8.429207 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.507819414e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=208912384 dispatch_meta_bytes=103512 combine_payload_bytes=208912384 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=209015896 ingress_bytes=3027484672 link_bytes=3027484672 token_bytes=417928280 weight_movement_bytes=2818572288 total_nvlink_bytes=3236500568 weight_to_token_ratio=6.744153 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.695265043e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32742 payload_rows_remote=19886 dup_rows=26 dup_rows_remote=0 dedup_ratio=0.000793 router_tokens_per_expert=[932, 127, 2356, 15, 2416, 3667, 6426, 445] payload_rows_per_dst=[1864, 254, 4712, 12856, 4832, 6390, 944, 890] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 0, 0] combine_rows_per_src=[1864, 254, 4712, 12856, 4832, 6390, 944, 890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32696 payload_rows_remote=25362 dup_rows=72 dup_rows_remote=0 dedup_ratio=0.002197 router_tokens_per_expert=[1594, 51, 2706, 32, 1240, 3248, 7263, 250] payload_rows_per_dst=[3188, 102, 1644, 64, 2480, 3358, 14526, 7334] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 72] combine_rows_per_src=[3188, 102, 1644, 64, 2480, 3358, 14526, 7334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201015296 dispatch_meta_bytes=99024 combine_payload_bytes=201015296 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201114320 ingress_bytes=1610301440 link_bytes=1610301440 token_bytes=402129616 weight_movement_bytes=1409286144 total_nvlink_bytes=1811415760 weight_to_token_ratio=3.504557 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.017015891e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32512 payload_rows_remote=26706 dup_rows=256 dup_rows_remote=256 dedup_ratio=0.007812 router_tokens_per_expert=[1368, 116, 2903, 745, 1607, 2273, 6955, 417] payload_rows_per_dst=[4564, 5172, 5806, 10260, 3214, 2662, 0, 834] dup_rows_per_dst=[56, 48, 0, 152, 0, 0, 0, 0] combine_rows_per_src=[4564, 5172, 5806, 10260, 3214, 2662, 0, 834] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=150896640 dispatch_meta_bytes=74120 combine_payload_bytes=150896640 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=150970760 ingress_bytes=5788041216 link_bytes=5788041216 token_bytes=301867400 weight_movement_bytes=5637144576 total_nvlink_bytes=5939011976 weight_to_token_ratio=18.674241 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.241061476e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=162906112 dispatch_meta_bytes=79544 combine_payload_bytes=162906112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=162985656 ingress_bytes=1572192256 link_bytes=1572192256 token_bytes=325891768 weight_movement_bytes=1409286144 total_nvlink_bytes=1735177912 weight_to_token_ratio=4.324399 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.803620368e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=207765504 dispatch_meta_bytes=101448 combine_payload_bytes=207765504 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=207866952 ingress_bytes=1617051648 link_bytes=1617051648 token_bytes=415632456 weight_movement_bytes=1409286144 total_nvlink_bytes=1824918600 weight_to_token_ratio=3.390703 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.054814238e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31320 payload_rows_remote=17958 dup_rows=1448 dup_rows_remote=930 dedup_ratio=0.044189 router_tokens_per_expert=[670, 1353, 3444, 4314, 96, 129, 6270, 108] payload_rows_per_dst=[13362, 7576, 8444, 1272, 192, 258, 0, 216] dup_rows_per_dst=[518, 712, 218, 0, 0, 0, 0, 0] combine_rows_per_src=[13362, 7576, 8444, 1272, 192, 258, 0, 216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=23150 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[150, 1739, 667, 7667, 0, 8, 5892, 261] payload_rows_per_dst=[300, 3478, 1334, 0, 8008, 9618, 5048, 4982] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[300, 3478, 1334, 0, 8008, 9618, 5048, 4982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31986 payload_rows_remote=25768 dup_rows=782 dup_rows_remote=66 dedup_ratio=0.023865 router_tokens_per_expert=[1527, 600, 2427, 41, 758, 3870, 7049, 112] payload_rows_per_dst=[3054, 1200, 3084, 8726, 6218, 4606, 1136, 3962] dup_rows_per_dst=[0, 0, 0, 36, 716, 24, 0, 6] combine_rows_per_src=[3054, 1200, 3084, 8726, 6218, 4606, 1136, 3962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=218775552 dispatch_meta_bytes=107848 combine_payload_bytes=218775552 prefetch_bytes=0 reduce_bytes=2818572288 egress_bytes=218883400 ingress_bytes=3037347840 link_bytes=3037347840 token_bytes=437658952 weight_movement_bytes=2818572288 total_nvlink_bytes=3256231240 weight_to_token_ratio=6.440111 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.700788005e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32644 payload_rows_remote=20032 dup_rows=124 dup_rows_remote=74 dedup_ratio=0.003784 router_tokens_per_expert=[1823, 59, 2846, 37, 200, 4015, 6272, 1132] payload_rows_per_dst=[5808, 12612, 5692, 74, 400, 5794, 0, 2264] dup_rows_per_dst=[74, 50, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5808, 12612, 5692, 74, 400, 5794, 0, 2264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=147111936 dispatch_meta_bytes=75552 combine_payload_bytes=147111936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=147187488 ingress_bytes=1556398080 link_bytes=1556398080 token_bytes=294299424 weight_movement_bytes=1409286144 total_nvlink_bytes=1703585568 weight_to_token_ratio=4.788613 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2176 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.715179575e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=189644800 dispatch_meta_bytes=92600 combine_payload_bytes=189644800 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=189737400 ingress_bytes=1598930944 link_bytes=1598930944 token_bytes=379382200 weight_movement_bytes=1409286144 total_nvlink_bytes=1788668344 weight_to_token_ratio=3.714687 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.953345860e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=25688 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000732 router_tokens_per_expert=[252, 2573, 523, 5851, 2, 66, 7113, 4] payload_rows_per_dst=[504, 5146, 1046, 0, 4, 5646, 7056, 13342] dup_rows_per_dst=[0, 0, 0, 0, 0, 22, 0, 2] combine_rows_per_src=[504, 5146, 1046, 0, 4, 5646, 7056, 13342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=211091456 dispatch_meta_bytes=103336 combine_payload_bytes=211091456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=211194792 ingress_bytes=1620377600 link_bytes=1620377600 token_bytes=422286248 weight_movement_bytes=1409286144 total_nvlink_bytes=1831572392 weight_to_token_ratio=3.337277 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5888 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.073438181e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=28606 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[65, 2285, 1500, 7648, 0, 4, 4841, 41] payload_rows_per_dst=[130, 4570, 3000, 4160, 11136, 5948, 3740, 82] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[130, 4570, 3000, 4160, 11136, 5948, 3740, 82] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=164102144 dispatch_meta_bytes=80424 combine_payload_bytes=164102144 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=164182568 ingress_bytes=1573388288 link_bytes=1573388288 token_bytes=328284712 weight_movement_bytes=1409286144 total_nvlink_bytes=1737570856 weight_to_token_ratio=4.292878 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.810317648e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32588 payload_rows_remote=18726 dup_rows=180 dup_rows_remote=4 dedup_ratio=0.005493 router_tokens_per_expert=[89, 1525, 1168, 6996, 73, 11, 6413, 109] payload_rows_per_dst=[178, 2888, 2388, 108, 146, 4684, 8334, 13862] dup_rows_per_dst=[0, 0, 2, 0, 0, 2, 0, 176] combine_rows_per_src=[178, 2888, 2388, 108, 146, 4684, 8334, 13862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=210436096 dispatch_meta_bytes=102848 combine_payload_bytes=210436096 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=210538944 ingress_bytes=4438294528 link_bytes=4438294528 token_bytes=420975040 weight_movement_bytes=4227858432 total_nvlink_bytes=4648833472 weight_to_token_ratio=10.043015 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.485259672e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30018 payload_rows_remote=26938 dup_rows=2750 dup_rows_remote=2750 dedup_ratio=0.083923 router_tokens_per_expert=[166, 1638, 1540, 7691, 1, 26, 5298, 24] payload_rows_per_dst=[1878, 3276, 3080, 15422, 2, 3910, 2402, 48] dup_rows_per_dst=[12, 0, 0, 2738, 0, 0, 0, 0] combine_rows_per_src=[1878, 3276, 3080, 15422, 2, 3910, 2402, 48] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114432 combine_payload_bytes=234340352 prefetch_bytes=1409286144 reduce_bytes=4932501504 egress_bytes=234454784 ingress_bytes=6576128000 link_bytes=6576128000 token_bytes=468795136 weight_movement_bytes=6341787648 total_nvlink_bytes=6810582784 weight_to_token_ratio=13.527844 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.682357179e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24358 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[19, 8149, 297, 24, 0, 7866, 29, 0] payload_rows_per_dst=[38, 0, 594, 48, 14658, 8410, 8046, 974] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[38, 0, 594, 48, 14658, 8410, 8046, 974] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32528 payload_rows_remote=17204 dup_rows=240 dup_rows_remote=0 dedup_ratio=0.007324 router_tokens_per_expert=[166, 7649, 40, 869, 21, 7616, 20, 3] payload_rows_per_dst=[15324, 15298, 80, 1738, 42, 0, 40, 6] dup_rows_per_dst=[240, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15324, 15298, 80, 1738, 42, 0, 40, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=153403392 dispatch_meta_bytes=74920 combine_payload_bytes=153403392 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=153478312 ingress_bytes=1562689536 link_bytes=1562689536 token_bytes=306881704 weight_movement_bytes=1409286144 total_nvlink_bytes=1716167848 weight_to_token_ratio=4.592278 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.750409103e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19488 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1391, 1515, 2501, 6639, 1, 16, 4308, 13] payload_rows_per_dst=[2782, 3030, 5002, 0, 13280, 2460, 6188, 26] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2782, 3030, 5002, 0, 13280, 2460, 6188, 26] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=220676096 dispatch_meta_bytes=118752 combine_payload_bytes=220676096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=220794848 ingress_bytes=1629962240 link_bytes=1629962240 token_bytes=441470944 weight_movement_bytes=1409286144 total_nvlink_bytes=1850757088 weight_to_token_ratio=3.192251 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.127108164e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=199540736 dispatch_meta_bytes=97432 combine_payload_bytes=199540736 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=199638168 ingress_bytes=5836685312 link_bytes=5836685312 token_bytes=399178904 weight_movement_bytes=5637144576 total_nvlink_bytes=6036323480 weight_to_token_ratio=14.121850 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.268300139e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31936 payload_rows_remote=29844 dup_rows=832 dup_rows_remote=832 dedup_ratio=0.025391 router_tokens_per_expert=[321, 1046, 1488, 7173, 0, 109, 6231, 16] payload_rows_per_dst=[12952, 2092, 7574, 9068, 0, 218, 0, 32] dup_rows_per_dst=[152, 0, 680, 0, 0, 0, 0, 0] combine_rows_per_src=[12952, 2092, 7574, 9068, 0, 218, 0, 32] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=140935168 dispatch_meta_bytes=68816 combine_payload_bytes=140935168 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=141003984 ingress_bytes=1550221312 link_bytes=1550221312 token_bytes=281939152 weight_movement_bytes=1409286144 total_nvlink_bytes=1691225296 weight_to_token_ratio=4.998547 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.680592253e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24454 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[97, 8176, 131, 26, 1, 7821, 132, 0] payload_rows_per_dst=[194, 0, 262, 52, 2, 7606, 8314, 16338] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[194, 0, 262, 52, 2, 7606, 8314, 16338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=159645696 dispatch_meta_bytes=77952 combine_payload_bytes=159645696 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=159723648 ingress_bytes=1568931840 link_bytes=1568931840 token_bytes=319369344 weight_movement_bytes=1409286144 total_nvlink_bytes=1728655488 weight_to_token_ratio=4.412716 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.785363400e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=19022 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[2, 8191, 1, 6, 0, 8183, 1, 0] payload_rows_per_dst=[4, 0, 16024, 13744, 2216, 776, 2, 0] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 16024, 13744, 2216, 776, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32722 payload_rows_remote=17292 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.001404 router_tokens_per_expert=[94, 8137, 192, 241, 3, 7690, 26, 1] payload_rows_per_dst=[188, 0, 384, 482, 6, 8038, 8194, 15430] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 46, 0] combine_rows_per_src=[188, 0, 384, 482, 6, 8038, 8194, 15430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=244482048 dispatch_meta_bytes=122704 combine_payload_bytes=244482048 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=244604752 ingress_bytes=2358411264 link_bytes=2358411264 token_bytes=489086800 weight_movement_bytes=2113929216 total_nvlink_bytes=2603016016 weight_to_token_ratio=4.322196 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.320611863e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=200327168 dispatch_meta_bytes=97816 combine_payload_bytes=200327168 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=200424984 ingress_bytes=1609613312 link_bytes=1609613312 token_bytes=400752152 weight_movement_bytes=1409286144 total_nvlink_bytes=1810038296 weight_to_token_ratio=3.516603 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.013162662e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32630 payload_rows_remote=17588 dup_rows=138 dup_rows_remote=58 dedup_ratio=0.004211 router_tokens_per_expert=[114, 8153, 40, 271, 0, 7752, 54, 0] payload_rows_per_dst=[318, 1264, 15042, 15898, 0, 0, 108, 0] dup_rows_per_dst=[56, 0, 80, 2, 0, 0, 0, 0] combine_rows_per_src=[318, 1264, 15042, 15898, 0, 0, 108, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=155828224 dispatch_meta_bytes=76096 combine_payload_bytes=155828224 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=155904320 ingress_bytes=1565114368 link_bytes=1565114368 token_bytes=311732544 weight_movement_bytes=1409286144 total_nvlink_bytes=1721018688 weight_to_token_ratio=4.520818 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1408 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.763987150e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32458 payload_rows_remote=26402 dup_rows=310 dup_rows_remote=310 dedup_ratio=0.009460 router_tokens_per_expert=[8, 1767, 6806, 29, 397, 2, 23, 7352] payload_rows_per_dst=[16, 3534, 0, 6176, 7978, 6056, 3818, 4880] dup_rows_per_dst=[0, 0, 0, 0, 310, 0, 0, 0] combine_rows_per_src=[16, 3534, 0, 6176, 7978, 6056, 3818, 4880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=141656064 dispatch_meta_bytes=69352 combine_payload_bytes=141656064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=141725416 ingress_bytes=1550942208 link_bytes=1550942208 token_bytes=283381480 weight_movement_bytes=1409286144 total_nvlink_bytes=1692667624 weight_to_token_ratio=4.973106 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.684628970e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32652 payload_rows_remote=21402 dup_rows=116 dup_rows_remote=112 dedup_ratio=0.003540 router_tokens_per_expert=[6, 3077, 5621, 189, 141, 555, 510, 6285] payload_rows_per_dst=[11250, 7646, 7336, 2698, 282, 2420, 1020, 0] dup_rows_per_dst=[4, 96, 0, 16, 0, 0, 0, 0] combine_rows_per_src=[11250, 7646, 7336, 2698, 282, 2420, 1020, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=144080896 dispatch_meta_bytes=70584 combine_payload_bytes=144080896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=144151480 ingress_bytes=1553367040 link_bytes=1553367040 token_bytes=288232376 weight_movement_bytes=1409286144 total_nvlink_bytes=1697518520 weight_to_token_ratio=4.889410 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.698207017e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=216285184 dispatch_meta_bytes=106848 combine_payload_bytes=216285184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=216392032 ingress_bytes=1625571328 link_bytes=1625571328 token_bytes=432677216 weight_movement_bytes=1409286144 total_nvlink_bytes=1841963360 weight_to_token_ratio=3.257130 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.102520890e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32754 payload_rows_remote=16920 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000427 router_tokens_per_expert=[65, 8163, 9, 334, 3, 7807, 3, 0] payload_rows_per_dst=[130, 0, 166, 668, 15834, 7938, 8018, 0] dup_rows_per_dst=[0, 0, 14, 0, 0, 0, 0, 0] combine_rows_per_src=[130, 0, 166, 668, 15834, 7938, 8018, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32732 payload_rows_remote=16526 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.001099 router_tokens_per_expert=[24, 8103, 60, 68, 5, 8114, 0, 10] payload_rows_per_dst=[16240, 16206, 120, 136, 10, 0, 0, 20] dup_rows_per_dst=[36, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16240, 16206, 120, 136, 10, 0, 0, 20] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=175325184 dispatch_meta_bytes=86056 combine_payload_bytes=175325184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=175411240 ingress_bytes=1584611328 link_bytes=1584611328 token_bytes=350736424 weight_movement_bytes=1409286144 total_nvlink_bytes=1760022568 weight_to_token_ratio=4.018078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.873161987e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32146 payload_rows_remote=24958 dup_rows=622 dup_rows_remote=620 dedup_ratio=0.018982 router_tokens_per_expert=[8, 978, 7320, 7, 588, 43, 5, 7435] payload_rows_per_dst=[16, 1956, 0, 520, 7510, 86, 7188, 14870] dup_rows_per_dst=[0, 0, 0, 6, 614, 0, 2, 0] combine_rows_per_src=[16, 1956, 0, 520, 7510, 86, 7188, 14870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 2, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=27950 dup_rows=4 dup_rows_remote=2 dedup_ratio=0.000122 router_tokens_per_expert=[9, 1314, 7981, 16, 56, 0, 40, 6968] payload_rows_per_dst=[18, 2628, 13682, 4814, 112, 7650, 3860, 0] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 2, 0] combine_rows_per_src=[18, 2628, 13682, 4814, 112, 7650, 3860, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=138608640 dispatch_meta_bytes=67736 combine_payload_bytes=138608640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=138676376 ingress_bytes=1547894784 link_bytes=1547894784 token_bytes=277285016 weight_movement_bytes=1409286144 total_nvlink_bytes=1686571160 weight_to_token_ratio=5.082446 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.667564667e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32746 payload_rows_remote=19728 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000671 router_tokens_per_expert=[4, 1837, 7765, 102, 27, 12, 128, 6509] payload_rows_per_dst=[8, 3674, 0, 204, 7440, 24, 8378, 13018] dup_rows_per_dst=[0, 0, 0, 0, 14, 0, 8, 0] combine_rows_per_src=[8, 3674, 0, 204, 7440, 24, 8378, 13018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=135380992 dispatch_meta_bytes=66248 combine_payload_bytes=135380992 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=135447240 ingress_bytes=5772525568 link_bytes=5772525568 token_bytes=270828232 weight_movement_bytes=5637144576 total_nvlink_bytes=5907972808 weight_to_token_ratio=20.814464 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.232373361e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=204455936 dispatch_meta_bytes=102312 combine_payload_bytes=204455936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=204558248 ingress_bytes=1613742080 link_bytes=1613742080 token_bytes=409014184 weight_movement_bytes=1409286144 total_nvlink_bytes=1818300328 weight_to_token_ratio=3.445568 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.036282039e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32752 payload_rows_remote=23052 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000488 router_tokens_per_expert=[9, 2991, 7775, 20, 658, 19, 22, 4890] payload_rows_per_dst=[5852, 5982, 9700, 3480, 1316, 6378, 44, 0] dup_rows_per_dst=[16, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5852, 5982, 9700, 3480, 1316, 6378, 44, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=29770 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000244 router_tokens_per_expert=[48, 2, 6702, 924, 7147, 1176, 135, 250] payload_rows_per_dst=[96, 4, 6372, 1848, 7548, 2990, 13402, 500] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8, 0] combine_rows_per_src=[96, 4, 6372, 1848, 7548, 2990, 13402, 500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=228966400 dispatch_meta_bytes=111808 combine_payload_bytes=228966400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=229078208 ingress_bytes=1638252544 link_bytes=1638252544 token_bytes=458044608 weight_movement_bytes=1409286144 total_nvlink_bytes=1867330752 weight_to_token_ratio=3.076744 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.173530406e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=161611776 dispatch_meta_bytes=79000 combine_payload_bytes=161611776 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=161690776 ingress_bytes=5798756352 link_bytes=5798756352 token_bytes=323302552 weight_movement_bytes=5637144576 total_nvlink_bytes=5960447128 weight_to_token_ratio=17.436128 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.247061505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 0, 0, 1, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32748 payload_rows_remote=16664 dup_rows=20 dup_rows_remote=0 dedup_ratio=0.000610 router_tokens_per_expert=[16, 5, 4339, 344, 8036, 792, 253, 2599] payload_rows_per_dst=[16084, 7884, 1034, 458, 0, 1584, 506, 5198] dup_rows_per_dst=[20, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16084, 7884, 1034, 458, 0, 1584, 506, 5198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=188841984 dispatch_meta_bytes=92272 combine_payload_bytes=188841984 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=188934256 ingress_bytes=5825986560 link_bytes=5825986560 token_bytes=377776240 weight_movement_bytes=5637144576 total_nvlink_bytes=6014920816 weight_to_token_ratio=14.921914 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6528 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.262309285e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=26500 dup_rows=12 dup_rows_remote=2 dedup_ratio=0.000366 router_tokens_per_expert=[9, 2563, 7408, 62, 347, 41, 3, 5951] payload_rows_per_dst=[18, 5126, 2050, 7318, 6256, 3962, 8026, 0] dup_rows_per_dst=[0, 0, 0, 0, 10, 0, 2, 0] combine_rows_per_src=[18, 5126, 2050, 7318, 6256, 3962, 8026, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=243875840 dispatch_meta_bytes=119112 combine_payload_bytes=243875840 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=243994952 ingress_bytes=1653161984 link_bytes=1653161984 token_bytes=487870792 weight_movement_bytes=1409286144 total_nvlink_bytes=1897156936 weight_to_token_ratio=2.888646 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3584 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.257017047e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32742 payload_rows_remote=30616 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000793 router_tokens_per_expert=[0, 1063, 7785, 44, 463, 57, 210, 6762] payload_rows_per_dst=[15570, 2126, 0, 7508, 926, 6192, 420, 0] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 0, 0] combine_rows_per_src=[15570, 2126, 0, 7508, 926, 6192, 420, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32450 payload_rows_remote=25160 dup_rows=318 dup_rows_remote=316 dedup_ratio=0.009705 router_tokens_per_expert=[10, 1, 5924, 1780, 7810, 126, 47, 686] payload_rows_per_dst=[20, 2, 0, 3560, 6148, 8206, 7290, 7224] dup_rows_per_dst=[0, 0, 0, 0, 0, 46, 2, 270] combine_rows_per_src=[20, 2, 0, 3560, 6148, 8206, 7290, 7224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=136511488 dispatch_meta_bytes=66656 combine_payload_bytes=136511488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=136578144 ingress_bytes=1545797632 link_bytes=1545797632 token_bytes=273089632 weight_movement_bytes=1409286144 total_nvlink_bytes=1682375776 weight_to_token_ratio=5.160526 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.655821492e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=23920 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 4525, 4424, 6954, 206, 0, 274] payload_rows_per_dst=[2, 7902, 9050, 8848, 6006, 412, 0, 548] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 7902, 9050, 8848, 6006, 412, 0, 548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=217088000 dispatch_meta_bytes=106008 combine_payload_bytes=217088000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=217194008 ingress_bytes=1626374144 link_bytes=1626374144 token_bytes=434282008 weight_movement_bytes=1409286144 total_nvlink_bytes=1843568152 weight_to_token_ratio=3.245094 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.107016324e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=250806272 dispatch_meta_bytes=122568 combine_payload_bytes=250806272 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=250928840 ingress_bytes=1660092416 link_bytes=1660092416 token_bytes=501735112 weight_movement_bytes=1409286144 total_nvlink_bytes=1911021256 weight_to_token_ratio=2.808825 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.295824573e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=206110720 dispatch_meta_bytes=101904 combine_payload_bytes=206110720 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=206212624 ingress_bytes=1615396864 link_bytes=1615396864 token_bytes=412323344 weight_movement_bytes=1409286144 total_nvlink_bytes=1821609488 weight_to_token_ratio=3.417915 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.045548139e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32380 payload_rows_remote=17274 dup_rows=388 dup_rows_remote=306 dedup_ratio=0.011841 router_tokens_per_expert=[105, 56, 7529, 499, 7827, 239, 69, 60] payload_rows_per_dst=[210, 112, 0, 998, 0, 15816, 138, 15106] dup_rows_per_dst=[0, 0, 0, 0, 0, 306, 0, 82] combine_rows_per_src=[210, 112, 0, 998, 0, 15816, 138, 15106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=20644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 3, 6225, 1245, 8108, 436, 0, 367] payload_rows_per_dst=[2334, 7120, 12124, 2490, 7094, 872, 0, 734] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2334, 7120, 12124, 2490, 7094, 872, 0, 734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=195952640 dispatch_meta_bytes=95680 combine_payload_bytes=195952640 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=196048320 ingress_bytes=1605238784 link_bytes=1605238784 token_bytes=392000960 weight_movement_bytes=1409286144 total_nvlink_bytes=1801287104 weight_to_token_ratio=3.595109 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.988667131e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31190 payload_rows_remote=30326 dup_rows=1578 dup_rows_remote=1570 dedup_ratio=0.048157 router_tokens_per_expert=[17, 2120, 0, 558, 7386, 134, 8, 6161] payload_rows_per_dst=[34, 1414, 0, 1116, 12608, 864, 11250, 3904] dup_rows_per_dst=[0, 0, 0, 0, 1570, 8, 0, 0] combine_rows_per_src=[34, 1414, 0, 1116, 12608, 864, 11250, 3904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=28042 payload_rows_remote=15168 dup_rows=4726 dup_rows_remote=22 dedup_ratio=0.144226 router_tokens_per_expert=[2477, 3535, 4, 296, 5570, 162, 64, 4276] payload_rows_per_dst=[12874, 3136, 7430, 1060, 3090, 324, 128, 0] dup_rows_per_dst=[4704, 0, 0, 22, 0, 0, 0, 0] combine_rows_per_src=[12874, 3136, 7430, 1060, 3090, 324, 128, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=141508608 dispatch_meta_bytes=70320 combine_payload_bytes=141508608 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=141578928 ingress_bytes=1550794752 link_bytes=1550794752 token_bytes=283087536 weight_movement_bytes=1409286144 total_nvlink_bytes=1692373680 weight_to_token_ratio=4.978270 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3968 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.683803278e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=169115648 dispatch_meta_bytes=82576 combine_payload_bytes=169115648 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=169198224 ingress_bytes=5101617152 link_bytes=5101617152 token_bytes=338313872 weight_movement_bytes=4932501504 total_nvlink_bytes=5270815376 weight_to_token_ratio=14.579661 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.856692653e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=26790 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000183 router_tokens_per_expert=[55, 0, 3855, 3541, 8092, 69, 0, 772] payload_rows_per_dst=[110, 1568, 722, 11292, 5972, 138, 11416, 1544] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[110, 1568, 722, 11292, 5972, 138, 11416, 1544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=248430592 dispatch_meta_bytes=127584 combine_payload_bytes=248430592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=248558176 ingress_bytes=1657716736 link_bytes=1657716736 token_bytes=496988768 weight_movement_bytes=1409286144 total_nvlink_bytes=1906274912 weight_to_token_ratio=2.835650 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1280 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.282521757e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24592 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[99, 0, 5821, 1634, 6847, 1199, 3, 781] payload_rows_per_dst=[13892, 8176, 3466, 3268, 0, 2398, 6, 1562] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13892, 8176, 3466, 3268, 0, 2398, 6, 1562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=25346 dup_rows=34 dup_rows_remote=32 dedup_ratio=0.001038 router_tokens_per_expert=[312, 2513, 0, 8, 7323, 48, 3, 6177] payload_rows_per_dst=[624, 4140, 0, 16, 102, 8036, 7388, 12428] dup_rows_per_dst=[0, 0, 0, 0, 0, 32, 2, 0] combine_rows_per_src=[624, 4140, 0, 16, 102, 8036, 7388, 12428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=124256256 dispatch_meta_bytes=60760 combine_payload_bytes=124256256 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=124317016 ingress_bytes=1533542400 link_bytes=1533542400 token_bytes=248573272 weight_movement_bytes=1409286144 total_nvlink_bytes=1657859416 weight_to_token_ratio=5.669500 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3200 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.587197308e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=23270 dup_rows=64 dup_rows_remote=36 dedup_ratio=0.001953 router_tokens_per_expert=[127, 799, 1, 129, 7899, 19, 0, 7410] payload_rows_per_dst=[254, 1598, 7448, 9434, 562, 7792, 5616, 0] dup_rows_per_dst=[0, 0, 0, 28, 0, 36, 0, 0] combine_rows_per_src=[254, 1598, 7448, 9434, 562, 7792, 5616, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=219463680 dispatch_meta_bytes=107184 combine_payload_bytes=219463680 prefetch_bytes=1409286144 reduce_bytes=4932501504 egress_bytes=219570864 ingress_bytes=6561251328 link_bytes=6561251328 token_bytes=439034544 weight_movement_bytes=6341787648 total_nvlink_bytes=6780822192 weight_to_token_ratio=14.444849 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7808 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.674026864e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=201457664 dispatch_meta_bytes=98368 combine_payload_bytes=201457664 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201556032 ingress_bytes=1610743808 link_bytes=1610743808 token_bytes=403013696 weight_movement_bytes=1409286144 total_nvlink_bytes=1812299840 weight_to_token_ratio=3.496869 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019492967e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32150 payload_rows_remote=16302 dup_rows=618 dup_rows_remote=30 dedup_ratio=0.018860 router_tokens_per_expert=[356, 3093, 0, 9, 5178, 256, 0, 7492] payload_rows_per_dst=[712, 4734, 0, 18, 0, 7632, 3206, 15848] dup_rows_per_dst=[0, 0, 0, 0, 0, 30, 0, 588] combine_rows_per_src=[712, 4734, 0, 18, 0, 7632, 3206, 15848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=207634432 dispatch_meta_bytes=101512 combine_payload_bytes=207634432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=207735944 ingress_bytes=1616920576 link_bytes=1616920576 token_bytes=415370376 weight_movement_bytes=1409286144 total_nvlink_bytes=1824656520 weight_to_token_ratio=3.392842 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.054080290e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32518 payload_rows_remote=24186 dup_rows=250 dup_rows_remote=250 dedup_ratio=0.007629 router_tokens_per_expert=[236, 4871, 0, 187, 3324, 519, 1, 7246] payload_rows_per_dst=[4240, 5972, 8332, 10298, 2134, 1540, 2, 0] dup_rows_per_dst=[2, 0, 0, 244, 0, 4, 0, 0] combine_rows_per_src=[4240, 5972, 8332, 10298, 2134, 1540, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=190627840 dispatch_meta_bytes=93224 combine_payload_bytes=190627840 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=190721064 ingress_bytes=1599913984 link_bytes=1599913984 token_bytes=381348904 weight_movement_bytes=1409286144 total_nvlink_bytes=1790635048 weight_to_token_ratio=3.695530 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1024 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.958850473e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32634 payload_rows_remote=26852 dup_rows=134 dup_rows_remote=134 dedup_ratio=0.004089 router_tokens_per_expert=[0, 553, 25, 3724, 6705, 29, 0, 5348] payload_rows_per_dst=[0, 3678, 50, 7448, 4980, 5782, 7794, 2902] dup_rows_per_dst=[0, 134, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 3678, 50, 7448, 4980, 5782, 7794, 2902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=133545984 dispatch_meta_bytes=65328 combine_payload_bytes=133545984 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=133611312 ingress_bytes=5066047488 link_bytes=5066047488 token_bytes=267157296 weight_movement_bytes=4932501504 total_nvlink_bytes=5199658800 weight_to_token_ratio=18.462911 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.836775126e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32448 payload_rows_remote=18862 dup_rows=320 dup_rows_remote=248 dedup_ratio=0.009766 router_tokens_per_expert=[220, 815, 700, 4143, 4336, 33, 371, 5766] payload_rows_per_dst=[13586, 4986, 9448, 0, 3620, 66, 742, 0] dup_rows_per_dst=[72, 10, 238, 0, 0, 0, 0, 0] combine_rows_per_src=[13586, 4986, 9448, 0, 3620, 66, 742, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=198131712 dispatch_meta_bytes=97744 combine_payload_bytes=198131712 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=198229456 ingress_bytes=1607417856 link_bytes=1607417856 token_bytes=396361168 weight_movement_bytes=1409286144 total_nvlink_bytes=1805647312 weight_to_token_ratio=3.555561 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.000869024e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32156 payload_rows_remote=25346 dup_rows=612 dup_rows_remote=206 dedup_ratio=0.018677 router_tokens_per_expert=[579, 4416, 4, 6, 5246, 538, 0, 5595] payload_rows_per_dst=[1158, 8142, 8, 4988, 6810, 5974, 5076, 0] dup_rows_per_dst=[0, 0, 0, 0, 406, 206, 0, 0] combine_rows_per_src=[1158, 8142, 8, 4988, 6810, 5974, 5076, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32612 payload_rows_remote=28980 dup_rows=156 dup_rows_remote=156 dedup_ratio=0.004761 router_tokens_per_expert=[201, 4695, 30, 36, 6211, 145, 50, 5016] payload_rows_per_dst=[8048, 3632, 9514, 5542, 5486, 290, 100, 0] dup_rows_per_dst=[118, 0, 36, 2, 0, 0, 0, 0] combine_rows_per_src=[8048, 3632, 9514, 5542, 5486, 290, 100, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=219971584 dispatch_meta_bytes=107944 combine_payload_bytes=219971584 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=220079528 ingress_bytes=1629257728 link_bytes=1629257728 token_bytes=440051112 weight_movement_bytes=1409286144 total_nvlink_bytes=1849337256 weight_to_token_ratio=3.202551 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.123163191e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=154517504 dispatch_meta_bytes=76440 combine_payload_bytes=154517504 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=154593944 ingress_bytes=1563803648 link_bytes=1563803648 token_bytes=309111448 weight_movement_bytes=1409286144 total_nvlink_bytes=1718397592 weight_to_token_ratio=4.559152 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.756647665e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32638 payload_rows_remote=32636 dup_rows=130 dup_rows_remote=130 dedup_ratio=0.003967 router_tokens_per_expert=[17, 518, 7, 4233, 4962, 12, 1, 6634] payload_rows_per_dst=[34, 1036, 14, 8256, 6880, 3068, 2, 13348] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 130] combine_rows_per_src=[34, 1036, 14, 8256, 6880, 3068, 2, 13348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=30072 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.001038 router_tokens_per_expert=[0, 1930, 43, 5958, 3302, 0, 65, 5086] payload_rows_per_dst=[0, 4946, 9312, 2662, 5516, 3464, 6834, 0] dup_rows_per_dst=[0, 2, 28, 0, 0, 0, 4, 0] combine_rows_per_src=[0, 4946, 9312, 2662, 5516, 3464, 6834, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=207634432 dispatch_meta_bytes=102208 combine_payload_bytes=207634432 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=207736640 ingress_bytes=5140135936 link_bytes=5140135936 token_bytes=415371072 weight_movement_bytes=4932501504 total_nvlink_bytes=5347872576 weight_to_token_ratio=11.874928 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.878261564e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31730 payload_rows_remote=16380 dup_rows=1038 dup_rows_remote=0 dedup_ratio=0.031677 router_tokens_per_expert=[52, 1335, 46, 5035, 5161, 5, 22, 4728] payload_rows_per_dst=[104, 2670, 92, 3138, 2798, 7534, 44, 15350] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 1038] combine_rows_per_src=[104, 2670, 92, 3138, 2798, 7534, 44, 15350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=237404160 dispatch_meta_bytes=116544 combine_payload_bytes=237404160 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=237520704 ingress_bytes=3055976448 link_bytes=3055976448 token_bytes=474924864 weight_movement_bytes=2818572288 total_nvlink_bytes=3293497152 weight_to_token_ratio=5.934775 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.711219248e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=267354112 dispatch_meta_bytes=131064 combine_payload_bytes=267354112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267485176 ingress_bytes=1676640256 link_bytes=1676640256 token_bytes=534839288 weight_movement_bytes=1409286144 total_nvlink_bytes=1944125432 weight_to_token_ratio=2.634971 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.388485570e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=246349824 dispatch_meta_bytes=120424 combine_payload_bytes=246349824 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=246470248 ingress_bytes=3769565184 link_bytes=3769565184 token_bytes=492820072 weight_movement_bytes=3523215360 total_nvlink_bytes=4016035432 weight_to_token_ratio=7.149091 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.110799154e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32480 payload_rows_remote=25532 dup_rows=288 dup_rows_remote=288 dedup_ratio=0.008789 router_tokens_per_expert=[0, 870, 23, 3451, 6392, 0, 3, 5645] payload_rows_per_dst=[8172, 2920, 6948, 0, 4978, 6502, 2960, 0] dup_rows_per_dst=[0, 288, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8172, 2920, 6948, 0, 4978, 6502, 2960, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134184960 dispatch_meta_bytes=65520 combine_payload_bytes=134184960 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=134250480 ingress_bytes=5066686464 link_bytes=5066686464 token_bytes=268435440 weight_movement_bytes=4932501504 total_nvlink_bytes=5200936944 weight_to_token_ratio=18.375001 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837132926e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31406 payload_rows_remote=17120 dup_rows=1362 dup_rows_remote=0 dedup_ratio=0.041565 router_tokens_per_expert=[0, 18, 14, 2560, 4491, 7457, 38, 1806] payload_rows_per_dst=[0, 36, 28, 5120, 2396, 14286, 5772, 3768] dup_rows_per_dst=[0, 0, 0, 0, 0, 1362, 0, 0] combine_rows_per_src=[0, 36, 28, 5120, 2396, 14286, 5772, 3768] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31594 payload_rows_remote=20164 dup_rows=1174 dup_rows_remote=1172 dedup_ratio=0.035828 router_tokens_per_expert=[2, 13, 1085, 3218, 5754, 5714, 118, 480] payload_rows_per_dst=[11430, 11526, 7274, 168, 0, 0, 236, 960] dup_rows_per_dst=[2, 8, 1164, 0, 0, 0, 0, 0] combine_rows_per_src=[11430, 11526, 7274, 168, 0, 0, 236, 960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=209158144 dispatch_meta_bytes=103280 combine_payload_bytes=209158144 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=209261424 ingress_bytes=1618444288 link_bytes=1618444288 token_bytes=418419568 weight_movement_bytes=1409286144 total_nvlink_bytes=1827705712 weight_to_token_ratio=3.368117 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.062612441e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32682 payload_rows_remote=31090 dup_rows=86 dup_rows_remote=86 dedup_ratio=0.002625 router_tokens_per_expert=[9, 604, 13, 5633, 2958, 7, 54, 7106] payload_rows_per_dst=[18, 5498, 792, 10500, 1592, 14, 14268, 0] dup_rows_per_dst=[0, 34, 0, 0, 0, 0, 52, 0] combine_rows_per_src=[18, 5498, 792, 10500, 1592, 14, 14268, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=140247040 dispatch_meta_bytes=68480 combine_payload_bytes=140247040 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=140315520 ingress_bytes=5777391616 link_bytes=5777391616 token_bytes=280562560 weight_movement_bytes=5637144576 total_nvlink_bytes=5917707136 weight_to_token_ratio=20.092291 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.235098145e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31976 payload_rows_remote=26188 dup_rows=792 dup_rows_remote=14 dedup_ratio=0.024170 router_tokens_per_expert=[18, 888, 49, 2874, 5973, 20, 34, 6528] payload_rows_per_dst=[10772, 5788, 5842, 764, 2404, 6338, 68, 0] dup_rows_per_dst=[10, 778, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[10772, 5788, 5842, 764, 2404, 6338, 68, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32522 payload_rows_remote=19518 dup_rows=246 dup_rows_remote=24 dedup_ratio=0.007507 router_tokens_per_expert=[1, 59, 45, 2713, 6142, 7256, 122, 46] payload_rows_per_dst=[2, 118, 90, 5426, 3632, 0, 13004, 10250] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 222, 24] combine_rows_per_src=[2, 118, 90, 5426, 3632, 0, 13004, 10250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=165183488 dispatch_meta_bytes=85344 combine_payload_bytes=165183488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=165268832 ingress_bytes=1574469632 link_bytes=1574469632 token_bytes=330452320 weight_movement_bytes=1409286144 total_nvlink_bytes=1739738464 weight_to_token_ratio=4.264719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.816372723e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 2, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32108 payload_rows_remote=24680 dup_rows=660 dup_rows_remote=490 dedup_ratio=0.020142 router_tokens_per_expert=[0, 2, 59, 3708, 4370, 8106, 48, 91] payload_rows_per_dst=[2546, 4, 8120, 7428, 11914, 1818, 96, 182] dup_rows_per_dst=[0, 0, 108, 170, 382, 0, 0, 0] combine_rows_per_src=[2546, 4, 8120, 7428, 11914, 1818, 96, 182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=254689280 dispatch_meta_bytes=124704 combine_payload_bytes=254689280 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=254813984 ingress_bytes=4482547712 link_bytes=4482547712 token_bytes=509503264 weight_movement_bytes=4227858432 total_nvlink_bytes=4737361696 weight_to_token_ratio=8.298001 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.510039608e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31990 payload_rows_remote=16432 dup_rows=778 dup_rows_remote=0 dedup_ratio=0.023743 router_tokens_per_expert=[1, 4, 106, 1661, 6722, 7379, 23, 488] payload_rows_per_dst=[2, 8, 212, 3322, 0, 0, 12888, 15558] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 778] combine_rows_per_src=[2, 8, 212, 3322, 0, 0, 12888, 15558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=214532096 dispatch_meta_bytes=104808 combine_payload_bytes=214532096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=214636904 ingress_bytes=1623818240 link_bytes=1623818240 token_bytes=429169000 weight_movement_bytes=1409286144 total_nvlink_bytes=1838455144 weight_to_token_ratio=3.283756 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4224 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.092704329e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=159891456 dispatch_meta_bytes=78168 combine_payload_bytes=159891456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=159969624 ingress_bytes=1569177600 link_bytes=1569177600 token_bytes=319861080 weight_movement_bytes=1409286144 total_nvlink_bytes=1729147224 weight_to_token_ratio=4.405932 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.786739553e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=23906 dup_rows=24 dup_rows_remote=14 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 26, 2516, 6043, 7444, 63, 292] payload_rows_per_dst=[7700, 6382, 8838, 3508, 5606, 0, 126, 584] dup_rows_per_dst=[0, 0, 10, 14, 0, 0, 0, 0] combine_rows_per_src=[7700, 6382, 8838, 3508, 5606, 0, 126, 584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=202178560 dispatch_meta_bytes=100680 combine_payload_bytes=202178560 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=202279240 ingress_bytes=2316107776 link_bytes=2316107776 token_bytes=404457800 weight_movement_bytes=2113929216 total_nvlink_bytes=2518387016 weight_to_token_ratio=5.226575 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.296923675e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134610944 dispatch_meta_bytes=65728 combine_payload_bytes=134610944 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134676672 ingress_bytes=1543897088 link_bytes=1543897088 token_bytes=269287616 weight_movement_bytes=1409286144 total_nvlink_bytes=1678573760 weight_to_token_ratio=5.233386 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2432 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645179238e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32698 payload_rows_remote=25378 dup_rows=70 dup_rows_remote=70 dedup_ratio=0.002136 router_tokens_per_expert=[4517, 1895, 7489, 1, 66, 0, 2236, 180] payload_rows_per_dst=[4018, 6706, 0, 2, 9680, 7320, 3578, 1394] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 66, 0] combine_rows_per_src=[4018, 6706, 0, 2, 9680, 7320, 3578, 1394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30188 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[909, 2926, 7927, 412, 1473, 3, 2713, 21] payload_rows_per_dst=[2580, 5852, 15854, 824, 2946, 6, 4664, 42] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2580, 5852, 15854, 824, 2946, 6, 4664, 42] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=23930 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[17, 11, 10, 3594, 4419, 7975, 29, 329] payload_rows_per_dst=[34, 22, 668, 7188, 8838, 15302, 58, 658] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[34, 22, 668, 7188, 8838, 15302, 58, 658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=195837952 dispatch_meta_bytes=95680 combine_payload_bytes=195837952 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=195933632 ingress_bytes=1605124096 link_bytes=1605124096 token_bytes=391771584 weight_movement_bytes=1409286144 total_nvlink_bytes=1801057728 weight_to_token_ratio=3.597214 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1280 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.988024926e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=207896576 dispatch_meta_bytes=101792 combine_payload_bytes=207896576 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=207998368 ingress_bytes=1617182720 link_bytes=1617182720 token_bytes=415894944 weight_movement_bytes=1409286144 total_nvlink_bytes=1825181088 weight_to_token_ratio=3.388563 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.055548187e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32750 payload_rows_remote=18088 dup_rows=18 dup_rows_remote=16 dedup_ratio=0.000549 router_tokens_per_expert=[11, 21, 152, 3188, 7311, 5515, 183, 3] payload_rows_per_dst=[11052, 14662, 6240, 424, 0, 0, 366, 6] dup_rows_per_dst=[0, 2, 16, 0, 0, 0, 0, 0] combine_rows_per_src=[11052, 14662, 6240, 424, 0, 0, 366, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=32362 dup_rows=58 dup_rows_remote=58 dedup_ratio=0.001770 router_tokens_per_expert=[3658, 3805, 8150, 0, 30, 0, 708, 33] payload_rows_per_dst=[0, 8978, 0, 0, 6006, 8188, 348, 9190] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 56] combine_rows_per_src=[0, 8978, 0, 0, 6006, 8188, 348, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=247300096 dispatch_meta_bytes=120752 combine_payload_bytes=247300096 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=247420848 ingress_bytes=3770515456 link_bytes=3770515456 token_bytes=494720944 weight_movement_bytes=3523215360 total_nvlink_bytes=4017936304 weight_to_token_ratio=7.121622 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.111331266e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16540 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[1560, 656, 8129, 0, 1048, 769, 4215, 7] payload_rows_per_dst=[3120, 1312, 0, 16226, 2096, 1568, 4318, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[3120, 1312, 0, 16226, 2096, 1568, 4318, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=196034560 dispatch_meta_bytes=95720 combine_payload_bytes=196034560 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=196130280 ingress_bytes=5128536064 link_bytes=5128536064 token_bytes=392164840 weight_movement_bytes=4932501504 total_nvlink_bytes=5324666344 weight_to_token_ratio=12.577623 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.871766120e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=19570 dup_rows=76 dup_rows_remote=16 dedup_ratio=0.002319 router_tokens_per_expert=[1057, 841, 8085, 20, 942, 24, 5359, 56] payload_rows_per_dst=[0, 3072, 366, 40, 2226, 7396, 6470, 13122] dup_rows_per_dst=[0, 14, 0, 0, 2, 0, 0, 60] combine_rows_per_src=[0, 3072, 366, 40, 2226, 7396, 6470, 13122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=148176896 dispatch_meta_bytes=72416 combine_payload_bytes=148176896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=148249312 ingress_bytes=1557463040 link_bytes=1557463040 token_bytes=296426208 weight_movement_bytes=1409286144 total_nvlink_bytes=1705712352 weight_to_token_ratio=4.754256 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.721142907e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=265109504 dispatch_meta_bytes=129680 combine_payload_bytes=265109504 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=265239184 ingress_bytes=3083681792 link_bytes=3083681792 token_bytes=530348688 weight_movement_bytes=2818572288 total_nvlink_bytes=3348920976 weight_to_token_ratio=5.314564 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.726733084e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=31474 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4433, 707, 8062, 0, 252, 2, 2927, 1] payload_rows_per_dst=[8866, 1414, 1294, 14830, 504, 4, 5854, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8866, 1414, 1294, 14830, 504, 4, 5854, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=135495680 dispatch_meta_bytes=66168 combine_payload_bytes=135495680 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135561848 ingress_bytes=1544781824 link_bytes=1544781824 token_bytes=271057528 weight_movement_bytes=1409286144 total_nvlink_bytes=1680343672 weight_to_token_ratio=5.199214 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.650133391e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=160317440 dispatch_meta_bytes=78344 combine_payload_bytes=160317440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=160395784 ingress_bytes=1569603584 link_bytes=1569603584 token_bytes=320713224 weight_movement_bytes=1409286144 total_nvlink_bytes=1729999368 weight_to_token_ratio=4.394225 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.789124886e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 2, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=26092 dup_rows=8 dup_rows_remote=4 dedup_ratio=0.000244 router_tokens_per_expert=[2, 5735, 3, 2638, 1, 42, 7394, 569] payload_rows_per_dst=[4, 2284, 9188, 5276, 2, 6668, 8200, 1138] dup_rows_per_dst=[0, 0, 4, 0, 0, 4, 0, 0] combine_rows_per_src=[4, 2284, 9188, 5276, 2, 6668, 8200, 1138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32436 payload_rows_remote=16950 dup_rows=332 dup_rows_remote=0 dedup_ratio=0.010132 router_tokens_per_expert=[769, 7140, 118, 19, 3, 198, 7931, 206] payload_rows_per_dst=[15486, 442, 236, 7474, 7990, 396, 22, 390] dup_rows_per_dst=[332, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15486, 442, 236, 7474, 7990, 396, 22, 390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=257835008 dispatch_meta_bytes=125896 combine_payload_bytes=257835008 prefetch_bytes=704643072 reduce_bytes=5637144576 egress_bytes=257960904 ingress_bytes=6599622656 link_bytes=6599622656 token_bytes=515795912 weight_movement_bytes=6341787648 total_nvlink_bytes=6857583560 weight_to_token_ratio=12.295149 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.695513206e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32220 payload_rows_remote=24368 dup_rows=548 dup_rows_remote=10 dedup_ratio=0.016724 router_tokens_per_expert=[4730, 386, 8038, 2, 384, 54, 2733, 57] payload_rows_per_dst=[4456, 5228, 0, 844, 7852, 8270, 918, 4652] dup_rows_per_dst=[0, 0, 0, 0, 538, 0, 0, 10] combine_rows_per_src=[4456, 5228, 0, 844, 7852, 8270, 918, 4652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32752 payload_rows_remote=32562 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000488 router_tokens_per_expert=[4307, 95, 7627, 1, 457, 7, 3833, 57] payload_rows_per_dst=[9712, 190, 15254, 2, 914, 14, 6552, 114] dup_rows_per_dst=[16, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[9712, 190, 15254, 2, 914, 14, 6552, 114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32220 payload_rows_remote=19594 dup_rows=548 dup_rows_remote=548 dedup_ratio=0.016724 router_tokens_per_expert=[21, 7642, 11, 401, 0, 892, 6313, 1104] payload_rows_per_dst=[42, 0, 11088, 802, 0, 1784, 12626, 5878] dup_rows_per_dst=[0, 0, 14, 0, 0, 0, 0, 534] combine_rows_per_src=[42, 0, 11088, 802, 0, 1784, 12626, 5878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=213745664 dispatch_meta_bytes=104384 combine_payload_bytes=213745664 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=213850048 ingress_bytes=1623031808 link_bytes=1623031808 token_bytes=427595712 weight_movement_bytes=1409286144 total_nvlink_bytes=1836881856 weight_to_token_ratio=3.295838 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1280 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.088300638e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=138854400 dispatch_meta_bytes=67800 combine_payload_bytes=138854400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=138922200 ingress_bytes=1548140544 link_bytes=1548140544 token_bytes=277776600 weight_movement_bytes=1409286144 total_nvlink_bytes=1687062744 weight_to_token_ratio=5.073452 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1408 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.668940821e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32424 payload_rows_remote=30188 dup_rows=344 dup_rows_remote=344 dedup_ratio=0.010498 router_tokens_per_expert=[6, 4433, 0, 2949, 0, 41, 5003, 3952] payload_rows_per_dst=[12, 12204, 0, 2236, 4486, 5582, 0, 7904] dup_rows_per_dst=[0, 324, 0, 0, 0, 20, 0, 0] combine_rows_per_src=[12, 12204, 0, 2236, 4486, 5582, 0, 7904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 2, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=199622656 dispatch_meta_bytes=97512 combine_payload_bytes=199622656 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=199720168 ingress_bytes=1608908800 link_bytes=1608908800 token_bytes=399342824 weight_movement_bytes=1409286144 total_nvlink_bytes=1808628968 weight_to_token_ratio=3.529013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3840 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.009217688e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32738 payload_rows_remote=27698 dup_rows=30 dup_rows_remote=2 dedup_ratio=0.000916 router_tokens_per_expert=[62, 5971, 7, 3791, 0, 27, 5960, 566] payload_rows_per_dst=[124, 0, 8018, 7582, 0, 54, 11920, 5040] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 28] combine_rows_per_src=[124, 0, 8018, 7582, 0, 54, 11920, 5040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=266747904 dispatch_meta_bytes=130312 combine_payload_bytes=266747904 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=266878216 ingress_bytes=1676034048 link_bytes=1676034048 token_bytes=533626120 weight_movement_bytes=1409286144 total_nvlink_bytes=1942912264 weight_to_token_ratio=2.640962 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6016 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.385091058e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=160514048 dispatch_meta_bytes=80568 combine_payload_bytes=160514048 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=160594616 ingress_bytes=5797658624 link_bytes=5797658624 token_bytes=321108664 weight_movement_bytes=5637144576 total_nvlink_bytes=5958253240 weight_to_token_ratio=17.555255 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.246446823e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32314 payload_rows_remote=32306 dup_rows=454 dup_rows_remote=454 dedup_ratio=0.013855 router_tokens_per_expert=[1, 7277, 4, 605, 8, 46, 6177, 2266] payload_rows_per_dst=[1140, 13838, 8, 372, 7996, 4428, 0, 4532] dup_rows_per_dst=[0, 416, 0, 0, 0, 38, 0, 0] combine_rows_per_src=[1140, 13838, 8, 372, 7996, 4428, 0, 4532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=247300096 dispatch_meta_bytes=122128 combine_payload_bytes=247300096 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=247422224 ingress_bytes=2361229312 link_bytes=2361229312 token_bytes=494722320 weight_movement_bytes=2113929216 total_nvlink_bytes=2608651536 weight_to_token_ratio=4.272961 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.322189852e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=226902016 dispatch_meta_bytes=110800 combine_payload_bytes=226902016 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=227012816 ingress_bytes=1636188160 link_bytes=1636188160 token_bytes=453914832 weight_movement_bytes=1409286144 total_nvlink_bytes=1863200976 weight_to_token_ratio=3.104737 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.161970717e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32754 payload_rows_remote=31324 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000427 router_tokens_per_expert=[4217, 1639, 1819, 5511, 121, 3055, 6, 16] payload_rows_per_dst=[4542, 3278, 7530, 0, 2598, 1430, 8664, 4712] dup_rows_per_dst=[0, 0, 0, 0, 14, 0, 0, 0] combine_rows_per_src=[4542, 3278, 7530, 0, 2598, 1430, 8664, 4712] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=264650752 dispatch_meta_bytes=131040 combine_payload_bytes=264650752 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=264781792 ingress_bytes=1673936896 link_bytes=1673936896 token_bytes=529432544 weight_movement_bytes=1409286144 total_nvlink_bytes=1938718688 weight_to_token_ratio=2.661880 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.373347882e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31766 payload_rows_remote=27570 dup_rows=1002 dup_rows_remote=918 dedup_ratio=0.030579 router_tokens_per_expert=[1216, 1423, 3927, 5065, 70, 4374, 13, 296] payload_rows_per_dst=[4196, 2938, 7912, 7214, 140, 8748, 26, 592] dup_rows_per_dst=[84, 0, 918, 0, 0, 0, 0, 0] combine_rows_per_src=[4196, 2938, 7912, 7214, 140, 8748, 26, 592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32686 payload_rows_remote=32658 dup_rows=82 dup_rows_remote=82 dedup_ratio=0.002502 router_tokens_per_expert=[456, 3541, 0, 3398, 14, 289, 6593, 2093] payload_rows_per_dst=[912, 2948, 4134, 6796, 28, 13682, 0, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 82, 0, 0] combine_rows_per_src=[912, 2948, 4134, 6796, 28, 13682, 0, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=24768 dup_rows=12 dup_rows_remote=10 dedup_ratio=0.000366 router_tokens_per_expert=[4408, 2639, 974, 7649, 14, 692, 8, 0] payload_rows_per_dst=[4840, 4592, 3478, 686, 28, 3694, 7988, 7450] dup_rows_per_dst=[0, 0, 10, 0, 0, 0, 2, 0] combine_rows_per_src=[4840, 4592, 3478, 686, 28, 3694, 7988, 7450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=256606208 dispatch_meta_bytes=125352 combine_payload_bytes=256606208 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=256731560 ingress_bytes=2370535424 link_bytes=2370535424 token_bytes=513337768 weight_movement_bytes=2113929216 total_nvlink_bytes=2627266984 weight_to_token_ratio=4.118008 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7680 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.327400886e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32262 payload_rows_remote=31954 dup_rows=506 dup_rows_remote=502 dedup_ratio=0.015442 router_tokens_per_expert=[463, 6956, 38, 262, 26, 15, 7055, 1569] payload_rows_per_dst=[14254, 308, 76, 2206, 12250, 30, 0, 3138] dup_rows_per_dst=[462, 4, 0, 24, 16, 0, 0, 0] combine_rows_per_src=[14254, 308, 76, 2206, 12250, 30, 0, 3138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=225853440 dispatch_meta_bytes=113952 combine_payload_bytes=225853440 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=225967392 ingress_bytes=3749068800 link_bytes=3749068800 token_bytes=451820832 weight_movement_bytes=3523215360 total_nvlink_bytes=3975036192 weight_to_token_ratio=7.797815 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.099322034e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30984 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4202, 1042, 183, 7696, 0, 3103, 3, 155] payload_rows_per_dst=[8404, 2084, 366, 1784, 13608, 2020, 6, 4496] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8404, 2084, 366, 1784, 13608, 2020, 6, 4496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=267534336 dispatch_meta_bytes=130960 combine_payload_bytes=267534336 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267665296 ingress_bytes=1676820480 link_bytes=1676820480 token_bytes=535199632 weight_movement_bytes=1409286144 total_nvlink_bytes=1944485776 weight_to_token_ratio=2.633197 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.389494749e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32686 payload_rows_remote=23154 dup_rows=82 dup_rows_remote=70 dedup_ratio=0.002502 router_tokens_per_expert=[3665, 2223, 637, 8062, 34, 1346, 27, 390] payload_rows_per_dst=[0, 3416, 4330, 1030, 68, 6142, 8168, 9532] dup_rows_per_dst=[0, 0, 40, 0, 0, 24, 6, 12] combine_rows_per_src=[0, 3416, 4330, 1030, 68, 6142, 8168, 9532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 2, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=202899456 dispatch_meta_bytes=99112 combine_payload_bytes=202899456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=202998568 ingress_bytes=1612185600 link_bytes=1612185600 token_bytes=405898024 weight_movement_bytes=1409286144 total_nvlink_bytes=1815184168 weight_to_token_ratio=3.472020 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.027566401e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=31342 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2973, 2870, 713, 7773, 2, 1568, 4, 481] payload_rows_per_dst=[5946, 5740, 1426, 7354, 8196, 2678, 8, 1420] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5946, 5740, 1426, 7354, 8196, 2678, 8, 1420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=261767168 dispatch_meta_bytes=129824 combine_payload_bytes=261767168 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=261896992 ingress_bytes=5194268672 link_bytes=5194268672 token_bytes=523664160 weight_movement_bytes=4932501504 total_nvlink_bytes=5456165664 weight_to_token_ratio=9.419208 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.908573637e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=253820928 dispatch_meta_bytes=123936 combine_payload_bytes=253820928 prefetch_bytes=704643072 reduce_bytes=5637144576 egress_bytes=253944864 ingress_bytes=6595608576 link_bytes=6595608576 token_bytes=507765792 weight_movement_bytes=6341787648 total_nvlink_bytes=6849553440 weight_to_token_ratio=12.489592 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.693265488e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=189677568 dispatch_meta_bytes=92896 combine_payload_bytes=189677568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=189770464 ingress_bytes=1598963712 link_bytes=1598963712 token_bytes=379448032 weight_movement_bytes=1409286144 total_nvlink_bytes=1788734176 weight_to_token_ratio=3.714043 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.953529347e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=25510 dup_rows=52 dup_rows_remote=50 dedup_ratio=0.001587 router_tokens_per_expert=[1327, 279, 4377, 6489, 3579, 22, 24, 287] payload_rows_per_dst=[130, 2206, 5614, 2646, 5138, 7206, 5740, 4036] dup_rows_per_dst=[0, 10, 0, 0, 0, 2, 0, 40] combine_rows_per_src=[130, 2206, 5614, 2646, 5138, 7206, 5740, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32166 payload_rows_remote=24282 dup_rows=602 dup_rows_remote=36 dedup_ratio=0.018372 router_tokens_per_expert=[3936, 1071, 2466, 7736, 33, 77, 41, 1024] payload_rows_per_dst=[7884, 9302, 4932, 7756, 8, 154, 82, 2048] dup_rows_per_dst=[566, 36, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7884, 9302, 4932, 7756, 8, 154, 82, 2048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=256753664 dispatch_meta_bytes=125368 combine_payload_bytes=256753664 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=256879032 ingress_bytes=1666039808 link_bytes=1666039808 token_bytes=513632696 weight_movement_bytes=1409286144 total_nvlink_bytes=1922918840 weight_to_token_ratio=2.743763 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7424 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.329127486e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32750 payload_rows_remote=25188 dup_rows=18 dup_rows_remote=4 dedup_ratio=0.000549 router_tokens_per_expert=[2788, 3599, 37, 7662, 25, 2214, 34, 25] payload_rows_per_dst=[2828, 7198, 2822, 0, 7562, 32, 7862, 4446] dup_rows_per_dst=[0, 0, 0, 0, 14, 0, 4, 0] combine_rows_per_src=[2828, 7198, 2822, 0, 7562, 32, 7862, 4446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31286 payload_rows_remote=18620 dup_rows=1482 dup_rows_remote=1482 dedup_ratio=0.045227 router_tokens_per_expert=[3920, 577, 946, 6331, 3188, 17, 2, 1403] payload_rows_per_dst=[1472, 1154, 3238, 4538, 78, 394, 12666, 7746] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 1482] combine_rows_per_src=[1472, 1154, 3238, 4538, 78, 394, 12666, 7746] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=208977920 dispatch_meta_bytes=102240 combine_payload_bytes=208977920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=209080160 ingress_bytes=1618264064 link_bytes=1618264064 token_bytes=418058080 weight_movement_bytes=1409286144 total_nvlink_bytes=1827344224 weight_to_token_ratio=3.371030 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.061603262e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=198918144 dispatch_meta_bytes=97272 combine_payload_bytes=198918144 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=199015416 ingress_bytes=3722133504 link_bytes=3722133504 token_bytes=397933560 weight_movement_bytes=3523215360 total_nvlink_bytes=3921148920 weight_to_token_ratio=8.853778 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.084239393e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29520 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5514, 937, 2898, 2112, 4886, 0, 0, 37] payload_rows_per_dst=[7948, 4720, 5712, 3248, 7010, 4056, 0, 74] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7948, 4720, 5712, 3248, 7010, 4056, 0, 74] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=29240 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000183 router_tokens_per_expert=[964, 1761, 1572, 8001, 15, 4000, 17, 54] payload_rows_per_dst=[1928, 3522, 3930, 14700, 540, 8000, 34, 108] dup_rows_per_dst=[0, 0, 6, 0, 0, 0, 0, 0] combine_rows_per_src=[1928, 3522, 3930, 14700, 540, 8000, 34, 108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=206340096 dispatch_meta_bytes=100768 combine_payload_bytes=206340096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=206440864 ingress_bytes=1615626240 link_bytes=1615626240 token_bytes=412780960 weight_movement_bytes=1409286144 total_nvlink_bytes=1822067104 weight_to_token_ratio=3.414126 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.046832548e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32748 payload_rows_remote=29454 dup_rows=20 dup_rows_remote=0 dedup_ratio=0.000610 router_tokens_per_expert=[3097, 393, 168, 7138, 3991, 5, 1, 1591] payload_rows_per_dst=[0, 786, 334, 0, 7852, 6204, 14278, 3294] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 20] combine_rows_per_src=[0, 786, 334, 0, 7852, 6204, 14278, 3294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=152535040 dispatch_meta_bytes=80408 combine_payload_bytes=152535040 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=152615448 ingress_bytes=1561821184 link_bytes=1561821184 token_bytes=305150488 weight_movement_bytes=1409286144 total_nvlink_bytes=1714436632 weight_to_token_ratio=4.618332 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.745546694e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32546 payload_rows_remote=22700 dup_rows=222 dup_rows_remote=102 dedup_ratio=0.006775 router_tokens_per_expert=[4090, 36, 3255, 7180, 1747, 0, 0, 76] payload_rows_per_dst=[3434, 3114, 9846, 4078, 2422, 9500, 0, 152] dup_rows_per_dst=[0, 6, 120, 0, 96, 0, 0, 0] combine_rows_per_src=[3434, 3114, 9846, 4078, 2422, 9500, 0, 152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=241827840 dispatch_meta_bytes=118080 combine_payload_bytes=241827840 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=241945920 ingress_bytes=5174329344 link_bytes=5174329344 token_bytes=483773760 weight_movement_bytes=4932501504 total_nvlink_bytes=5416275264 weight_to_token_ratio=10.195885 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.897408446e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=239534080 dispatch_meta_bytes=116984 combine_payload_bytes=239534080 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=239651064 ingress_bytes=2353463296 link_bytes=2353463296 token_bytes=479185144 weight_movement_bytes=2113929216 total_nvlink_bytes=2593114360 weight_to_token_ratio=4.411508 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.317841207e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=241287168 dispatch_meta_bytes=117816 combine_payload_bytes=241287168 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=241404984 ingress_bytes=1650573312 link_bytes=1650573312 token_bytes=482692152 weight_movement_bytes=1409286144 total_nvlink_bytes=1891978296 weight_to_token_ratio=2.919638 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6784 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.242521564e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24382 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[783, 50, 198, 4607, 11, 8183, 1831, 721] payload_rows_per_dst=[1566, 100, 5718, 2658, 6740, 8386, 5144, 2456] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[1566, 100, 5718, 2658, 6740, 8386, 5144, 2456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32014 payload_rows_remote=16018 dup_rows=754 dup_rows_remote=0 dedup_ratio=0.023010 router_tokens_per_expert=[429, 0, 275, 5348, 50, 7963, 2293, 26] payload_rows_per_dst=[15996, 8124, 550, 2606, 100, 0, 4586, 52] dup_rows_per_dst=[754, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15996, 8124, 550, 2606, 100, 0, 4586, 52] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=23900 dup_rows=92 dup_rows_remote=92 dedup_ratio=0.002808 router_tokens_per_expert=[3915, 211, 2294, 5696, 3094, 125, 0, 1049] payload_rows_per_dst=[6538, 940, 1356, 7684, 8776, 5244, 0, 2138] dup_rows_per_dst=[0, 86, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[6538, 940, 1356, 7684, 8776, 5244, 0, 2138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=185958400 dispatch_meta_bytes=91208 combine_payload_bytes=185958400 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=186049608 ingress_bytes=3004530688 link_bytes=3004530688 token_bytes=372008008 weight_movement_bytes=2818572288 total_nvlink_bytes=3190580296 weight_to_token_ratio=7.576644 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7552 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.682411770e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 2, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=199737344 dispatch_meta_bytes=97528 combine_payload_bytes=199737344 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=199834872 ingress_bytes=5836881920 link_bytes=5836881920 token_bytes=399572216 weight_movement_bytes=5637144576 total_nvlink_bytes=6036716792 weight_to_token_ratio=14.107949 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.268410232e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26098 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[60, 8, 34, 8001, 16, 8149, 106, 10] payload_rows_per_dst=[120, 16, 68, 0, 1588, 11360, 6670, 12946] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[120, 16, 68, 0, 1588, 11360, 6670, 12946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=131219456 dispatch_meta_bytes=64072 combine_payload_bytes=131219456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=131283528 ingress_bytes=1540505600 link_bytes=1540505600 token_bytes=262502984 weight_movement_bytes=1409286144 total_nvlink_bytes=1671789128 weight_to_token_ratio=5.368648 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3072 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.626188321e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29440 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2457, 304, 0, 5418, 9, 8140, 10, 46] payload_rows_per_dst=[4914, 6578, 8804, 3328, 9032, 0, 20, 92] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4914, 6578, 8804, 3328, 9032, 0, 20, 92] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=195788800 dispatch_meta_bytes=95968 combine_payload_bytes=195788800 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=195884768 ingress_bytes=3014361088 link_bytes=3014361088 token_bytes=391673568 weight_movement_bytes=2818572288 total_nvlink_bytes=3210245856 weight_to_token_ratio=7.196228 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7168 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.687916384e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=22358 dup_rows=84 dup_rows_remote=2 dedup_ratio=0.002563 router_tokens_per_expert=[2398, 350, 689, 4217, 2859, 1, 1, 5869] payload_rows_per_dst=[4796, 10326, 1614, 2818, 1388, 2, 2, 11738] dup_rows_per_dst=[0, 82, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[4796, 10326, 1614, 2818, 1388, 2, 2, 11738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32530 payload_rows_remote=18634 dup_rows=238 dup_rows_remote=0 dedup_ratio=0.007263 router_tokens_per_expert=[440, 8, 29, 6754, 31, 8129, 838, 155] payload_rows_per_dst=[880, 16, 58, 0, 62, 8070, 9548, 13896] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 238] combine_rows_per_src=[880, 16, 58, 0, 62, 8070, 9548, 13896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=213794816 dispatch_meta_bytes=104392 combine_payload_bytes=213794816 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=213899208 ingress_bytes=1623080960 link_bytes=1623080960 token_bytes=427694024 weight_movement_bytes=1409286144 total_nvlink_bytes=1836980168 weight_to_token_ratio=3.295080 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5120 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.088575869e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6797697312 prefetch_bytes=20434649088 reduce_bytes=18320719872 weight_movement_bytes=38755368960 egress_bytes=3399688480 ingress_bytes=42153377792 link_bytes=42153377792 total_nvlink_bytes=45553066272 weight_to_token_ratio=5.701250 prefetch_share_of_ingress=0.484769 dedup_ratio=0.013512 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=26 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.475258250e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=241172480 dispatch_meta_bytes=117760 combine_payload_bytes=241172480 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=241290240 ingress_bytes=5878317056 link_bytes=5878317056 token_bytes=482462720 weight_movement_bytes=5637144576 total_nvlink_bytes=6119607296 weight_to_token_ratio=11.684104 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7424 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.291612178e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=5578010720 prefetch_bytes=19730006016 reduce_bytes=11274289152 weight_movement_bytes=31004295168 egress_bytes=2789699680 ingress_bytes=33792606208 link_bytes=33792606208 total_nvlink_bytes=36582305888 weight_to_token_ratio=5.558307 prefetch_share_of_ingress=0.583856 dedup_ratio=0.028069 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=16 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.182653056e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5, 0, 0, 8118, 17, 8188, 33, 23] payload_rows_per_dst=[10, 8184, 8192, 11432, 4838, 0, 66, 46] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10, 8184, 8192, 11432, 4838, 0, 66, 46] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=183156736 dispatch_meta_bytes=89440 combine_payload_bytes=183156736 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=183246176 ingress_bytes=1592442880 link_bytes=1592442880 token_bytes=366402912 weight_movement_bytes=1409286144 total_nvlink_bytes=1775689056 weight_to_token_ratio=3.846274 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=2048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.917015410e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=152649728 dispatch_meta_bytes=74536 combine_payload_bytes=152649728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=152724264 ingress_bytes=1561935872 link_bytes=1561935872 token_bytes=305373992 weight_movement_bytes=1409286144 total_nvlink_bytes=1714660136 weight_to_token_ratio=4.614951 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1664 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.746188899e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6326275280 prefetch_bytes=19730006016 reduce_bytes=21139292160 weight_movement_bytes=40869298176 egress_bytes=3163917520 ingress_bytes=44031655936 link_bytes=44031655936 total_nvlink_bytes=47195573456 weight_to_token_ratio=6.460247 prefetch_share_of_ingress=0.448087 dedup_ratio=0.008316 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=30 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.540993085e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=23080 dup_rows=48 dup_rows_remote=0 dedup_ratio=0.001465 router_tokens_per_expert=[9, 18, 106, 7047, 163, 6673, 880, 1488] payload_rows_per_dst=[18, 36, 8402, 204, 9640, 4952, 6492, 2976] dup_rows_per_dst=[0, 0, 0, 0, 48, 0, 0, 0] combine_rows_per_src=[18, 36, 8402, 204, 9640, 4952, 6492, 2976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6730691272 prefetch_bytes=19730006016 reduce_bytes=28890365952 weight_movement_bytes=48620371968 egress_bytes=3366171336 ingress_bytes=51984891904 link_bytes=51984891904 total_nvlink_bytes=55351063240 weight_to_token_ratio=7.223682 prefetch_share_of_ingress=0.379533 dedup_ratio=0.004768 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=41 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.819335594e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=5404210928 prefetch_bytes=19025362944 reduce_bytes=21139292160 weight_movement_bytes=40164655104 egress_bytes=2702767856 ingress_bytes=42866098176 link_bytes=42866098176 total_nvlink_bytes=45568866032 weight_to_token_ratio=7.432104 prefetch_share_of_ingress=0.443832 dedup_ratio=0.011860 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=30 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8001 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.500201604e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6749648096 prefetch_bytes=19730006016 reduce_bytes=19730006016 weight_movement_bytes=39460012032 egress_bytes=3375658208 ingress_bytes=42834001920 link_bytes=42834001920 total_nvlink_bytes=46209660128 weight_to_token_ratio=5.846232 prefetch_share_of_ingress=0.460616 dedup_ratio=0.011337 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=28 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8001 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.499078318e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=189071360 dispatch_meta_bytes=92320 combine_payload_bytes=189071360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=189163680 ingress_bytes=1598357504 link_bytes=1598357504 token_bytes=378235040 weight_movement_bytes=1409286144 total_nvlink_bytes=1787521184 weight_to_token_ratio=3.725953 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.950134835e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32636 payload_rows_remote=22924 dup_rows=132 dup_rows_remote=130 dedup_ratio=0.004028 router_tokens_per_expert=[248, 9, 227, 7379, 360, 8007, 121, 33] payload_rows_per_dst=[8380, 9712, 976, 12540, 720, 0, 242, 66] dup_rows_per_dst=[130, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8380, 9712, 976, 12540, 720, 0, 242, 66] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6551337904 prefetch_bytes=20434649088 reduce_bytes=21843935232 weight_movement_bytes=42278584320 egress_bytes=3276471216 ingress_bytes=45553451008 link_bytes=45553451008 total_nvlink_bytes=48829922224 weight_to_token_ratio=6.453428 prefetch_share_of_ingress=0.448586 dedup_ratio=0.006710 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=31 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.594251942e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=187793408 dispatch_meta_bytes=92216 combine_payload_bytes=187793408 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=187885624 ingress_bytes=1597079552 link_bytes=1597079552 token_bytes=375679032 weight_movement_bytes=1409286144 total_nvlink_bytes=1784965176 weight_to_token_ratio=3.751304 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.942978837e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6642787208 prefetch_bytes=19025362944 reduce_bytes=15502147584 weight_movement_bytes=34527510528 egress_bytes=3322209160 ingress_bytes=37848088576 link_bytes=37848088576 total_nvlink_bytes=41170297736 weight_to_token_ratio=5.197744 prefetch_share_of_ingress=0.502677 dedup_ratio=0.008396 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=22 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.324584359e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none - [2026-09-16 21:34:40] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 603.1 | throughput per GPU (TFLOP/s/GPU): 552.8 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 9.680272E+00 | loss scale: 1.0 | grad norm: 42.220 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32686 payload_rows_remote=31356 dup_rows=82 dup_rows_remote=82 dedup_ratio=0.002502 router_tokens_per_expert=[90, 136, 665, 1070, 2345, 2713, 1618, 7747] payload_rows_per_dst=[1150, 14716, 1330, 2366, 4462, 5426, 3236, 0] dup_rows_per_dst=[20, 60, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[1150, 14716, 1330, 2366, 4462, 5426, 3236, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32632 payload_rows_remote=17148 dup_rows=136 dup_rows_remote=0 dedup_ratio=0.004150 router_tokens_per_expert=[91, 52, 739, 1244, 2356, 2068, 2097, 7737] payload_rows_per_dst=[182, 104, 1478, 2488, 4712, 4136, 4048, 15484] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 136] combine_rows_per_src=[182, 104, 1478, 2488, 4712, 4136, 4048, 15484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31728 payload_rows_remote=25286 dup_rows=1040 dup_rows_remote=1040 dedup_ratio=0.031738 router_tokens_per_expert=[56, 24, 652, 1493, 3221, 1512, 1811, 7615] payload_rows_per_dst=[112, 1498, 14044, 2986, 6442, 3024, 3622, 0] dup_rows_per_dst=[0, 14, 1026, 0, 0, 0, 0, 0] combine_rows_per_src=[112, 1498, 14044, 2986, 6442, 3024, 3622, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30874 payload_rows_remote=26062 dup_rows=1894 dup_rows_remote=1486 dedup_ratio=0.057800 router_tokens_per_expert=[158, 101, 634, 1186, 2920, 1292, 2388, 7705] payload_rows_per_dst=[316, 202, 8628, 2372, 6302, 4812, 6716, 1526] dup_rows_per_dst=[0, 0, 596, 0, 144, 408, 746, 0] combine_rows_per_src=[316, 202, 8628, 2372, 6302, 4812, 6716, 1526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=11 export_slots=11 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 2, 2, 0, 2, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32570 payload_rows_remote=32424 dup_rows=198 dup_rows_remote=198 dedup_ratio=0.006042 router_tokens_per_expert=[59, 73, 1045, 1466, 2880, 1753, 1477, 7631] payload_rows_per_dst=[15294, 146, 2090, 8092, 488, 3506, 2954, 0] dup_rows_per_dst=[86, 0, 0, 112, 0, 0, 0, 0] combine_rows_per_src=[15294, 146, 2090, 8092, 488, 3506, 2954, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32452 payload_rows_remote=17304 dup_rows=316 dup_rows_remote=176 dedup_ratio=0.009644 router_tokens_per_expert=[89, 86, 923, 1616, 2260, 2031, 1824, 7555] payload_rows_per_dst=[15148, 172, 1846, 7576, 0, 4062, 3648, 0] dup_rows_per_dst=[140, 0, 0, 176, 0, 0, 0, 0] combine_rows_per_src=[15148, 172, 1846, 7576, 0, 4062, 3648, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=140476416 dispatch_meta_bytes=68592 combine_payload_bytes=140476416 prefetch_bytes=704643072 reduce_bytes=7751073792 egress_bytes=140545008 ingress_bytes=8596193280 link_bytes=8596193280 token_bytes=281021424 weight_movement_bytes=8455716864 total_nvlink_bytes=8736738288 weight_to_token_ratio=30.089225 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=4.813509414e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=256868352 dispatch_meta_bytes=125752 combine_payload_bytes=256868352 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=256994104 ingress_bytes=1666154496 link_bytes=1666154496 token_bytes=513862456 weight_movement_bytes=1409286144 total_nvlink_bytes=1923148600 weight_to_token_ratio=2.742536 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2944 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.329769690e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=207142912 dispatch_meta_bytes=105304 combine_payload_bytes=207142912 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=207248216 ingress_bytes=3025715200 link_bytes=3025715200 token_bytes=414391128 weight_movement_bytes=2818572288 total_nvlink_bytes=3232963416 weight_to_token_ratio=6.801720 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.694274212e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=213499904 dispatch_meta_bytes=110192 combine_payload_bytes=213499904 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=213610096 ingress_bytes=1622786048 link_bytes=1622786048 token_bytes=427110000 weight_movement_bytes=1409286144 total_nvlink_bytes=1836396144 weight_to_token_ratio=3.299586 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.086924485e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28262 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[75, 39, 378, 1447, 2607, 1774, 2253, 7811] payload_rows_per_dst=[150, 78, 756, 2894, 5214, 3548, 4506, 15622] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[150, 78, 756, 2894, 5214, 3548, 4506, 15622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=265617408 dispatch_meta_bytes=130488 combine_payload_bytes=265617408 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=265747896 ingress_bytes=1674903552 link_bytes=1674903552 token_bytes=531365304 weight_movement_bytes=1409286144 total_nvlink_bytes=1940651448 weight_to_token_ratio=2.652198 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.378760752e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32712 payload_rows_remote=29008 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.001709 router_tokens_per_expert=[85, 34, 487, 1852, 2502, 1923, 1646, 7855] payload_rows_per_dst=[170, 15722, 974, 3704, 5004, 3846, 3292, 0] dup_rows_per_dst=[0, 56, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[170, 15722, 974, 3704, 5004, 3846, 3292, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=141754368 dispatch_meta_bytes=69920 combine_payload_bytes=141754368 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=141824288 ingress_bytes=1551040512 link_bytes=1551040512 token_bytes=283578656 weight_movement_bytes=1409286144 total_nvlink_bytes=1692864800 weight_to_token_ratio=4.969648 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.685179431e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30024 payload_rows_remote=13808 dup_rows=2744 dup_rows_remote=132 dedup_ratio=0.083740 router_tokens_per_expert=[6723, 14, 16, 0, 1809, 131, 86, 7605] payload_rows_per_dst=[0, 28, 32, 0, 0, 13576, 172, 16216] dup_rows_per_dst=[0, 0, 0, 0, 0, 132, 0, 2612] combine_rows_per_src=[0, 28, 32, 0, 0, 13576, 172, 16216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=28492 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000732 router_tokens_per_expert=[6548, 0, 13, 0, 2126, 70, 16, 7611] payload_rows_per_dst=[0, 13096, 26, 4040, 4252, 140, 11190, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 24, 0] combine_rows_per_src=[0, 13096, 26, 4040, 4252, 140, 11190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32638 payload_rows_remote=30136 dup_rows=130 dup_rows_remote=0 dedup_ratio=0.003967 router_tokens_per_expert=[5951, 18, 81, 0, 2243, 84, 51, 7956] payload_rows_per_dst=[6210, 5728, 2502, 13442, 4486, 168, 102, 0] dup_rows_per_dst=[0, 0, 130, 0, 0, 0, 0, 0] combine_rows_per_src=[6210, 5728, 2502, 13442, 4486, 168, 102, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=231522304 dispatch_meta_bytes=113048 combine_payload_bytes=231522304 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=231635352 ingress_bytes=1640808448 link_bytes=1640808448 token_bytes=463157656 weight_movement_bytes=1409286144 total_nvlink_bytes=1872443800 weight_to_token_ratio=3.042778 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.187842402e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32282 payload_rows_remote=26892 dup_rows=486 dup_rows_remote=486 dedup_ratio=0.014832 router_tokens_per_expert=[5100, 0, 4, 3, 2997, 54, 234, 7992] payload_rows_per_dst=[0, 2768, 8, 6, 8098, 5390, 16012, 0] dup_rows_per_dst=[0, 0, 0, 0, 46, 0, 440, 0] combine_rows_per_src=[0, 2768, 8, 6, 8098, 5390, 16012, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=113115136 dispatch_meta_bytes=55760 combine_payload_bytes=113115136 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=113170896 ingress_bytes=5750259712 link_bytes=5750259712 token_bytes=226286032 weight_movement_bytes=5637144576 total_nvlink_bytes=5863430608 weight_to_token_ratio=24.911589 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6784 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.219905411e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233406464 dispatch_meta_bytes=114064 combine_payload_bytes=233406464 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233520528 ingress_bytes=3051978752 link_bytes=3051978752 token_bytes=466926992 weight_movement_bytes=2818572288 total_nvlink_bytes=3285499280 weight_to_token_ratio=6.036430 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7680 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708980705e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=246874112 dispatch_meta_bytes=120544 combine_payload_bytes=246874112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=246994656 ingress_bytes=1656160256 link_bytes=1656160256 token_bytes=493868768 weight_movement_bytes=1409286144 total_nvlink_bytes=1903154912 weight_to_token_ratio=2.853564 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.273806118e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=237633536 dispatch_meta_bytes=116256 combine_payload_bytes=237633536 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=237749792 ingress_bytes=1646919680 link_bytes=1646919680 token_bytes=475383328 weight_movement_bytes=1409286144 total_nvlink_bytes=1884669472 weight_to_token_ratio=2.964526 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5888 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.222062750e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=32760 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000244 router_tokens_per_expert=[6591, 0, 5, 1, 1611, 114, 128, 7934] payload_rows_per_dst=[13182, 0, 15870, 2, 3222, 228, 256, 0] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[13182, 0, 15870, 2, 3222, 228, 256, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=19390 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[6688, 4, 1, 0, 2463, 24, 116, 7088] payload_rows_per_dst=[13376, 8, 14176, 0, 4926, 48, 232, 0] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[13376, 8, 14176, 0, 4926, 48, 232, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=220299264 dispatch_meta_bytes=109512 combine_payload_bytes=220299264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=220408776 ingress_bytes=1629585408 link_bytes=1629585408 token_bytes=440708040 weight_movement_bytes=1409286144 total_nvlink_bytes=1849994184 weight_to_token_ratio=3.197777 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.124998062e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131072 combine_payload_bytes=268369920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268500992 ingress_bytes=1677656064 link_bytes=1677656064 token_bytes=536870912 weight_movement_bytes=1409286144 total_nvlink_bytes=1946157056 weight_to_token_ratio=2.625000 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394173670e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=158842880 dispatch_meta_bytes=77568 combine_payload_bytes=158842880 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=158920448 ingress_bytes=4386701312 link_bytes=4386701312 token_bytes=317763328 weight_movement_bytes=4227858432 total_nvlink_bytes=4545621760 weight_to_token_ratio=13.305055 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.456369625e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16510 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8129, 555, 1, 539, 6, 162, 6992, 0] payload_rows_per_dst=[0, 1110, 2, 1078, 12, 324, 13984, 16258] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 1110, 2, 1078, 12, 324, 13984, 16258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30552 payload_rows_remote=26352 dup_rows=2216 dup_rows_remote=2204 dedup_ratio=0.067627 router_tokens_per_expert=[6465, 0, 4, 1, 2139, 68, 11, 7696] payload_rows_per_dst=[0, 0, 8, 2, 1540, 13024, 4200, 11778] dup_rows_per_dst=[0, 0, 0, 0, 0, 42, 12, 2162] combine_rows_per_src=[0, 0, 8, 2, 1540, 13024, 4200, 11778] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30900 payload_rows_remote=17808 dup_rows=1868 dup_rows_remote=1868 dedup_ratio=0.057007 router_tokens_per_expert=[8179, 386, 8, 287, 0, 978, 6546, 0] payload_rows_per_dst=[0, 772, 318, 574, 13092, 16144, 0, 0] dup_rows_per_dst=[0, 0, 10, 0, 0, 1858, 0, 0] combine_rows_per_src=[0, 772, 318, 574, 13092, 16144, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32654 payload_rows_remote=16646 dup_rows=114 dup_rows_remote=114 dedup_ratio=0.003479 router_tokens_per_expert=[8116, 287, 9, 373, 0, 27, 7571, 1] payload_rows_per_dst=[242, 574, 16008, 15774, 0, 54, 0, 2] dup_rows_per_dst=[0, 0, 0, 114, 0, 0, 0, 0] combine_rows_per_src=[242, 574, 16008, 15774, 0, 54, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=135249920 dispatch_meta_bytes=66040 combine_payload_bytes=135249920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135315960 ingress_bytes=1544536064 link_bytes=1544536064 token_bytes=270565880 weight_movement_bytes=1409286144 total_nvlink_bytes=1679852024 weight_to_token_ratio=5.208662 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.648757237e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=215875584 dispatch_meta_bytes=114224 combine_payload_bytes=215875584 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=215989808 ingress_bytes=1625161728 link_bytes=1625161728 token_bytes=431865392 weight_movement_bytes=1409286144 total_nvlink_bytes=1841151536 weight_to_token_ratio=3.263253 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.100227301e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31838 payload_rows_remote=20538 dup_rows=930 dup_rows_remote=148 dedup_ratio=0.028381 router_tokens_per_expert=[8173, 259, 42, 349, 5, 610, 6946, 0] payload_rows_per_dst=[0, 518, 84, 698, 13742, 11300, 5366, 130] dup_rows_per_dst=[0, 0, 0, 0, 0, 782, 148, 0] combine_rows_per_src=[0, 518, 84, 698, 13742, 11300, 5366, 130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=145883136 dispatch_meta_bytes=78704 combine_payload_bytes=145883136 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=145961840 ingress_bytes=1555169280 link_bytes=1555169280 token_bytes=291844976 weight_movement_bytes=1409286144 total_nvlink_bytes=1701131120 weight_to_token_ratio=4.828886 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.708298808e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=136364032 dispatch_meta_bytes=67040 combine_payload_bytes=136364032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=136431072 ingress_bytes=1545650176 link_bytes=1545650176 token_bytes=272795104 weight_movement_bytes=1409286144 total_nvlink_bytes=1682081248 weight_to_token_ratio=5.166098 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.654995799e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17492 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5570, 0, 3, 1, 3099, 10, 64, 7637] payload_rows_per_dst=[0, 11140, 6, 15276, 6198, 20, 128, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 11140, 6, 15276, 6198, 20, 128, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=19480 dup_rows=12 dup_rows_remote=0 dedup_ratio=0.000366 router_tokens_per_expert=[8151, 164, 6, 121, 11, 284, 7647, 0] payload_rows_per_dst=[16302, 13276, 12, 2576, 22, 568, 0, 0] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16302, 13276, 12, 2576, 22, 568, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=16492 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.001587 router_tokens_per_expert=[8112, 871, 15, 204, 13, 601, 6565, 3] payload_rows_per_dst=[16224, 14820, 30, 408, 26, 1202, 0, 6] dup_rows_per_dst=[0, 52, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16224, 14820, 30, 408, 26, 1202, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=168247296 dispatch_meta_bytes=82744 combine_payload_bytes=168247296 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=168330040 ingress_bytes=1577533440 link_bytes=1577533440 token_bytes=336577336 weight_movement_bytes=1409286144 total_nvlink_bytes=1745863480 weight_to_token_ratio=4.187109 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1792 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.833528769e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16904 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1706, 16, 5144, 1, 5, 12, 1568, 7932] payload_rows_per_dst=[3412, 32, 0, 5106, 10, 5208, 3136, 15864] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3412, 32, 0, 5106, 10, 5208, 3136, 15864] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=143294464 dispatch_meta_bytes=69968 combine_payload_bytes=143294464 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=143364432 ingress_bytes=1552580608 link_bytes=1552580608 token_bytes=286658896 weight_movement_bytes=1409286144 total_nvlink_bytes=1695945040 weight_to_token_ratio=4.916248 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.693803326e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=159580160 dispatch_meta_bytes=77920 combine_payload_bytes=159580160 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=159658080 ingress_bytes=1568866304 link_bytes=1568866304 token_bytes=319238240 weight_movement_bytes=1409286144 total_nvlink_bytes=1728524384 weight_to_token_ratio=4.414528 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.784996425e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=135102464 dispatch_meta_bytes=66176 combine_payload_bytes=135102464 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=135168640 ingress_bytes=5772247040 link_bytes=5772247040 token_bytes=270271104 weight_movement_bytes=5637144576 total_nvlink_bytes=5907415680 weight_to_token_ratio=20.857371 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.232217397e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 1, 2, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19498 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8185, 472, 0, 52, 0, 210, 7465, 0] payload_rows_per_dst=[0, 944, 0, 104, 1660, 420, 13270, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 944, 0, 104, 1660, 420, 13270, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25372 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1272, 26, 6179, 1, 2, 0, 1169, 7735] payload_rows_per_dst=[2544, 52, 12358, 2, 7396, 8078, 2338, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2544, 52, 12358, 2, 7396, 8078, 2338, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=29376 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[1554, 4, 5895, 9, 2, 8, 1011, 7901] payload_rows_per_dst=[3108, 8408, 3390, 6714, 8110, 1014, 2022, 0] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[3108, 8408, 3390, 6714, 8110, 1014, 2022, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=138477568 dispatch_meta_bytes=67616 combine_payload_bytes=138477568 prefetch_bytes=0 reduce_bytes=7046430720 egress_bytes=138545184 ingress_bytes=7184908288 link_bytes=7184908288 token_bytes=277022752 weight_movement_bytes=7046430720 total_nvlink_bytes=7323453472 weight_to_token_ratio=25.436289 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=4.023248728e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32094 payload_rows_remote=25284 dup_rows=674 dup_rows_remote=674 dedup_ratio=0.020569 router_tokens_per_expert=[1750, 7, 7097, 0, 3, 0, 1355, 6172] payload_rows_per_dst=[3500, 14, 1414, 5994, 96, 6810, 13218, 1048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 674, 0] combine_rows_per_src=[3500, 14, 1414, 5994, 96, 6810, 13218, 1048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=207847424 dispatch_meta_bytes=101488 combine_payload_bytes=207847424 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=207948912 ingress_bytes=1617133568 link_bytes=1617133568 token_bytes=415796336 weight_movement_bytes=1409286144 total_nvlink_bytes=1825082480 weight_to_token_ratio=3.389366 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.055272956e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=240648192 dispatch_meta_bytes=117512 combine_payload_bytes=240648192 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=240765704 ingress_bytes=4468506624 link_bytes=4468506624 token_bytes=481413896 weight_movement_bytes=4227858432 total_nvlink_bytes=4709272328 weight_to_token_ratio=8.782169 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7680 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.502177185e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=159727616 dispatch_meta_bytes=77992 combine_payload_bytes=159727616 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=159805608 ingress_bytes=5796872192 link_bytes=5796872192 token_bytes=319533224 weight_movement_bytes=5637144576 total_nvlink_bytes=5956677800 weight_to_token_ratio=17.641810 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=7040 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.246006454e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=21324 dup_rows=12 dup_rows_remote=2 dedup_ratio=0.000366 router_tokens_per_expert=[8150, 345, 2, 180, 0, 58, 7648, 1] payload_rows_per_dst=[0, 690, 16302, 11432, 4214, 116, 0, 2] dup_rows_per_dst=[0, 0, 2, 10, 0, 0, 0, 0] combine_rows_per_src=[0, 690, 16302, 11432, 4214, 116, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31666 payload_rows_remote=19996 dup_rows=1102 dup_rows_remote=2 dedup_ratio=0.033630 router_tokens_per_expert=[1179, 9, 6432, 14, 1, 3, 1095, 7651] payload_rows_per_dst=[11670, 12880, 2122, 1722, 1076, 6, 2190, 0] dup_rows_per_dst=[1100, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[11670, 12880, 2122, 1722, 1076, 6, 2190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=21476 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[896, 2, 5643, 3, 3, 1, 1799, 8037] payload_rows_per_dst=[1792, 11290, 0, 7962, 8122, 2, 3598, 0] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[1792, 11290, 0, 7962, 8122, 2, 3598, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=207126528 dispatch_meta_bytes=103832 combine_payload_bytes=207126528 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=207230360 ingress_bytes=1616412672 link_bytes=1616412672 token_bytes=414356888 weight_movement_bytes=1409286144 total_nvlink_bytes=1823643032 weight_to_token_ratio=3.401141 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.051236239e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32310 payload_rows_remote=15968 dup_rows=458 dup_rows_remote=0 dedup_ratio=0.013977 router_tokens_per_expert=[170, 4126, 4, 0, 8, 3879, 26, 8171] payload_rows_per_dst=[340, 4468, 3792, 0, 16, 3888, 3464, 16342] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 458] combine_rows_per_src=[340, 4468, 3792, 0, 16, 3888, 3464, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=163807232 dispatch_meta_bytes=79992 combine_payload_bytes=163807232 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=163887224 ingress_bytes=1573093376 link_bytes=1573093376 token_bytes=327694456 weight_movement_bytes=1409286144 total_nvlink_bytes=1736980600 weight_to_token_ratio=4.300610 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5760 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.808666264e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=174686208 dispatch_meta_bytes=85304 combine_payload_bytes=174686208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=174771512 ingress_bytes=1583972352 link_bytes=1583972352 token_bytes=349457720 weight_movement_bytes=1409286144 total_nvlink_bytes=1758743864 weight_to_token_ratio=4.032780 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1280 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.869583989e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=175931392 dispatch_meta_bytes=85912 combine_payload_bytes=175931392 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=176017304 ingress_bytes=1585217536 link_bytes=1585217536 token_bytes=351948696 weight_movement_bytes=1409286144 total_nvlink_bytes=1761234840 weight_to_token_ratio=4.004237 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.876556499e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=24542 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[146, 4376, 0, 0, 2, 3687, 0, 8173] payload_rows_per_dst=[2872, 2226, 3946, 112, 8222, 3156, 12230, 0] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[2872, 2226, 3946, 112, 8222, 3156, 12230, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=32678 dup_rows=8 dup_rows_remote=0 dedup_ratio=0.000244 router_tokens_per_expert=[305, 3442, 5, 3, 0, 4486, 8, 8135] payload_rows_per_dst=[7986, 3584, 82, 8154, 8042, 4896, 16, 0] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[7986, 3584, 82, 8154, 8042, 4896, 16, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=28260 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[1356, 11, 4829, 6, 1, 1, 2252, 7928] payload_rows_per_dst=[2712, 22, 0, 5264, 2, 4404, 4504, 15856] dup_rows_per_dst=[0, 0, 0, 2, 0, 2, 0, 0] combine_rows_per_src=[2712, 22, 0, 5264, 2, 4404, 4504, 15856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=130809856 dispatch_meta_bytes=63872 combine_payload_bytes=130809856 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=130873728 ingress_bytes=5767954432 link_bytes=5767954432 token_bytes=261683584 weight_movement_bytes=5637144576 total_nvlink_bytes=5898828160 weight_to_token_ratio=21.541835 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.229813716e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201048064 dispatch_meta_bytes=98168 combine_payload_bytes=201048064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201146232 ingress_bytes=1610334208 link_bytes=1610334208 token_bytes=402194296 weight_movement_bytes=1409286144 total_nvlink_bytes=1811480440 weight_to_token_ratio=3.503993 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.017199378e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32690 payload_rows_remote=30150 dup_rows=78 dup_rows_remote=78 dedup_ratio=0.002380 router_tokens_per_expert=[147, 4244, 73, 1, 0, 3787, 0, 8132] payload_rows_per_dst=[294, 3136, 5420, 2, 8176, 2540, 13122, 0] dup_rows_per_dst=[0, 0, 78, 0, 0, 0, 0, 0] combine_rows_per_src=[294, 3136, 5420, 2, 8176, 2540, 13122, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=267698176 dispatch_meta_bytes=130712 combine_payload_bytes=267698176 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267828888 ingress_bytes=1676984320 link_bytes=1676984320 token_bytes=535527064 weight_movement_bytes=1409286144 total_nvlink_bytes=1944813208 weight_to_token_ratio=2.631587 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.390412184e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=231505920 dispatch_meta_bytes=113056 combine_payload_bytes=231505920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231618976 ingress_bytes=1640792064 link_bytes=1640792064 token_bytes=463124896 weight_movement_bytes=1409286144 total_nvlink_bytes=1872411040 weight_to_token_ratio=3.042994 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5504 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.187750658e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=22956 dup_rows=52 dup_rows_remote=48 dedup_ratio=0.001587 router_tokens_per_expert=[80, 2569, 33, 4, 10, 5549, 8, 8131] payload_rows_per_dst=[9760, 4820, 8170, 8118, 20, 1812, 16, 0] dup_rows_per_dst=[4, 0, 46, 2, 0, 0, 0, 0] combine_rows_per_src=[9760, 4820, 8170, 8118, 20, 1812, 16, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=27566 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[174, 3846, 1, 0, 1, 4228, 1, 8133] payload_rows_per_dst=[5672, 5200, 8078, 8190, 2, 5622, 2, 0] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5672, 5200, 8078, 8190, 2, 5622, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1465, 34, 6742, 0, 2, 1, 544, 7596] payload_rows_per_dst=[2930, 68, 13484, 0, 7954, 7244, 1088, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2930, 68, 13484, 0, 7954, 7244, 1088, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=246988800 dispatch_meta_bytes=120912 combine_payload_bytes=246988800 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=247109712 ingress_bytes=4474847232 link_bytes=4474847232 token_bytes=494098512 weight_movement_bytes=4227858432 total_nvlink_bytes=4721956944 weight_to_token_ratio=8.556712 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.505727661e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16444 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[7881, 10, 120, 8157, 8, 2, 202, 4] payload_rows_per_dst=[0, 20, 240, 0, 16, 8100, 8068, 16322] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[0, 20, 240, 0, 16, 8100, 8068, 16322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=188055552 dispatch_meta_bytes=92016 combine_payload_bytes=188055552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=188147568 ingress_bytes=1597341696 link_bytes=1597341696 token_bytes=376203120 weight_movement_bytes=1409286144 total_nvlink_bytes=1785489264 weight_to_token_ratio=3.746078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.944446734e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32606 payload_rows_remote=22418 dup_rows=162 dup_rows_remote=162 dedup_ratio=0.004944 router_tokens_per_expert=[7933, 2, 167, 8155, 0, 3, 124, 0] payload_rows_per_dst=[0, 1876, 6908, 0, 10188, 7418, 6216, 0] dup_rows_per_dst=[0, 2, 6, 0, 0, 0, 154, 0] combine_rows_per_src=[0, 1876, 6908, 0, 10188, 7418, 6216, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=225820672 dispatch_meta_bytes=110272 combine_payload_bytes=225820672 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=225930944 ingress_bytes=3044392960 link_bytes=3044392960 token_bytes=451751616 weight_movement_bytes=2818572288 total_nvlink_bytes=3270323904 weight_to_token_ratio=6.239208 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.704732978e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=31514 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[6996, 5, 627, 8116, 0, 0, 640, 0] payload_rows_per_dst=[0, 14002, 1254, 10054, 6178, 0, 1280, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 14002, 1254, 10054, 6178, 0, 1280, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29224 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[219, 3919, 2, 0, 0, 4084, 9, 8151] payload_rows_per_dst=[438, 4696, 3146, 0, 126, 4850, 3544, 15968] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[438, 4696, 3146, 0, 126, 4850, 3544, 15968] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134709248 dispatch_meta_bytes=65784 combine_payload_bytes=134709248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134775032 ingress_bytes=1543995392 link_bytes=1543995392 token_bytes=269484280 weight_movement_bytes=1409286144 total_nvlink_bytes=1678770424 weight_to_token_ratio=5.229567 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645729700e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=183648256 dispatch_meta_bytes=90320 combine_payload_bytes=183648256 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=183738576 ingress_bytes=1592934400 link_bytes=1592934400 token_bytes=367386832 weight_movement_bytes=1409286144 total_nvlink_bytes=1776672976 weight_to_token_ratio=3.835973 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.919767716e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=258162688 dispatch_meta_bytes=126056 combine_payload_bytes=258162688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=258288744 ingress_bytes=1667448832 link_bytes=1667448832 token_bytes=516451432 weight_movement_bytes=1409286144 total_nvlink_bytes=1925737576 weight_to_token_ratio=2.728787 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.337017432e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31994 payload_rows_remote=24240 dup_rows=774 dup_rows_remote=774 dedup_ratio=0.023621 router_tokens_per_expert=[7300, 2, 459, 8129, 0, 3, 491, 0] payload_rows_per_dst=[0, 4, 7770, 0, 7978, 7754, 8356, 132] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 774, 0] combine_rows_per_src=[0, 4, 7770, 0, 7978, 7754, 8356, 132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=239403008 dispatch_meta_bytes=116896 combine_payload_bytes=239403008 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=239519904 ingress_bytes=1648689152 link_bytes=1648689152 token_bytes=478922912 weight_movement_bytes=1409286144 total_nvlink_bytes=1888209056 weight_to_token_ratio=2.942616 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.231971054e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32658 payload_rows_remote=31602 dup_rows=110 dup_rows_remote=110 dedup_ratio=0.003357 router_tokens_per_expert=[7380, 1, 583, 8189, 0, 0, 231, 0] payload_rows_per_dst=[13706, 1056, 8910, 8524, 0, 0, 462, 0] dup_rows_per_dst=[0, 0, 110, 0, 0, 0, 0, 0] combine_rows_per_src=[13706, 1056, 8910, 8524, 0, 0, 462, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=29500 payload_rows_remote=13536 dup_rows=3268 dup_rows_remote=170 dedup_ratio=0.099731 router_tokens_per_expert=[7864, 11, 204, 8114, 7, 0, 184, 0] payload_rows_per_dst=[15964, 22, 6848, 6284, 14, 0, 368, 0] dup_rows_per_dst=[3098, 0, 170, 0, 0, 0, 0, 0] combine_rows_per_src=[15964, 22, 6848, 6284, 14, 0, 368, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24576 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[121, 3952, 1, 1, 0, 4134, 1, 8174] payload_rows_per_dst=[5400, 4636, 2, 8190, 8160, 6004, 374, 0] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5400, 4636, 2, 8190, 8160, 6004, 374, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=198574080 dispatch_meta_bytes=100056 combine_payload_bytes=198574080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=198674136 ingress_bytes=1607860224 link_bytes=1607860224 token_bytes=397248216 weight_movement_bytes=1409286144 total_nvlink_bytes=1806534360 weight_to_token_ratio=3.547621 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.003346101e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16482 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[3, 47, 1, 8144, 1, 17, 8171, 0] payload_rows_per_dst=[6, 94, 2, 0, 2, 8214, 8164, 16282] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 0, 0] combine_rows_per_src=[6, 94, 2, 0, 2, 8214, 8164, 16282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32758 payload_rows_remote=24622 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000305 router_tokens_per_expert=[0, 1, 2, 8180, 0, 11, 8190, 0] payload_rows_per_dst=[0, 2, 12, 8232, 8136, 8192, 8184, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 10, 0, 0] combine_rows_per_src=[0, 2, 12, 8232, 8136, 8192, 8184, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32754 payload_rows_remote=24566 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000427 router_tokens_per_expert=[0, 9, 0, 8186, 4, 0, 8185, 0] payload_rows_per_dst=[22, 8238, 8188, 8128, 8178, 0, 0, 0] dup_rows_per_dst=[0, 12, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[22, 8238, 8188, 8128, 8178, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=110886912 dispatch_meta_bytes=54824 combine_payload_bytes=110886912 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=110941736 ingress_bytes=5748031488 link_bytes=5748031488 token_bytes=221828648 weight_movement_bytes=5637144576 total_nvlink_bytes=5858973224 weight_to_token_ratio=25.412158 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7552 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.218657699e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=258883584 dispatch_meta_bytes=126848 combine_payload_bytes=258883584 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=259010432 ingress_bytes=1668169728 link_bytes=1668169728 token_bytes=517894016 weight_movement_bytes=1409286144 total_nvlink_bytes=1927180160 weight_to_token_ratio=2.721186 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.341054148e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98312 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424904 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751496 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037640 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=26116 dup_rows=60 dup_rows_remote=0 dedup_ratio=0.001831 router_tokens_per_expert=[7916, 1, 155, 8156, 0, 0, 156, 0] payload_rows_per_dst=[0, 2, 310, 0, 0, 9492, 6592, 16312] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 60, 0] combine_rows_per_src=[0, 2, 310, 0, 0, 9492, 6592, 16312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=135020544 dispatch_meta_bytes=65944 combine_payload_bytes=135020544 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135086488 ingress_bytes=1544306688 link_bytes=1544306688 token_bytes=270107032 weight_movement_bytes=1409286144 total_nvlink_bytes=1679393176 weight_to_token_ratio=5.217510 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.647472827e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201703424 dispatch_meta_bytes=98528 combine_payload_bytes=201703424 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201801952 ingress_bytes=1610989568 link_bytes=1610989568 token_bytes=403505376 weight_movement_bytes=1409286144 total_nvlink_bytes=1812791520 weight_to_token_ratio=3.492608 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.020869121e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201244672 dispatch_meta_bytes=98320 combine_payload_bytes=201244672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201342992 ingress_bytes=1610530816 link_bytes=1610530816 token_bytes=402587664 weight_movement_bytes=1409286144 total_nvlink_bytes=1811873808 weight_to_token_ratio=3.500570 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018300301e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24616 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 14, 0, 8183, 0, 0, 8187, 0] payload_rows_per_dst=[0, 28, 0, 8082, 8188, 8152, 8234, 84] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 28, 0, 8082, 8188, 8152, 8234, 84] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=213942272 dispatch_meta_bytes=104464 combine_payload_bytes=213942272 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=214046736 ingress_bytes=1623228416 link_bytes=1623228416 token_bytes=427989008 weight_movement_bytes=1409286144 total_nvlink_bytes=1837275152 weight_to_token_ratio=3.292809 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1280 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.089401561e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24570 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[1, 6, 0, 8188, 3, 1, 8184, 1] payload_rows_per_dst=[16378, 8196, 8182, 0, 6, 2, 0, 2] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 8196, 8182, 0, 6, 2, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201654272 dispatch_meta_bytes=98464 combine_payload_bytes=201654272 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201752736 ingress_bytes=1610940416 link_bytes=1610940416 token_bytes=403407008 weight_movement_bytes=1409286144 total_nvlink_bytes=1812693152 weight_to_token_ratio=3.493460 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.020593890e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24862 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7892, 0, 121, 8150, 0, 2, 218, 1] payload_rows_per_dst=[0, 15784, 242, 7906, 8394, 4, 436, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 15784, 242, 7906, 8394, 4, 436, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16408 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[20, 4, 1, 8144, 33, 0, 8172, 10] payload_rows_per_dst=[16360, 8130, 8192, 0, 66, 0, 0, 20] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16360, 8130, 8192, 0, 66, 0, 0, 20] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32426 payload_rows_remote=27102 dup_rows=342 dup_rows_remote=98 dedup_ratio=0.010437 router_tokens_per_expert=[2, 691, 1, 8169, 5205, 49, 12, 2255] payload_rows_per_dst=[4, 1382, 2, 0, 0, 15292, 10422, 5324] dup_rows_per_dst=[0, 0, 0, 0, 0, 86, 12, 244] combine_rows_per_src=[4, 1382, 2, 0, 0, 15292, 10422, 5324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 2, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1884, 0, 8183, 4769, 126, 3, 1419] payload_rows_per_dst=[0, 3768, 38, 16328, 5108, 252, 4436, 2838] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 3768, 38, 16328, 5108, 252, 4436, 2838] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201277440 dispatch_meta_bytes=98280 combine_payload_bytes=201277440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201375720 ingress_bytes=1610563584 link_bytes=1610563584 token_bytes=402653160 weight_movement_bytes=1409286144 total_nvlink_bytes=1811939304 weight_to_token_ratio=3.500000 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018483788e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16436 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1134, 0, 8190, 4069, 95, 0, 2896] payload_rows_per_dst=[48, 2268, 16332, 0, 8138, 190, 0, 5792] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[48, 2268, 16332, 0, 8138, 190, 0, 5792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4, 0, 8191, 0, 1, 8188, 0] payload_rows_per_dst=[0, 8, 0, 0, 0, 8194, 8186, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8, 0, 0, 0, 8194, 8186, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=203669504 dispatch_meta_bytes=99448 combine_payload_bytes=203669504 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=203768952 ingress_bytes=5840814080 link_bytes=5840814080 token_bytes=407438456 weight_movement_bytes=5637144576 total_nvlink_bytes=6044583032 weight_to_token_ratio=13.835573 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.270612077e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134414336 dispatch_meta_bytes=65632 combine_payload_bytes=134414336 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134479968 ingress_bytes=1543700480 link_bytes=1543700480 token_bytes=268894304 weight_movement_bytes=1409286144 total_nvlink_bytes=1678180448 weight_to_token_ratio=5.241041 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644078316e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=222019584 dispatch_meta_bytes=108800 combine_payload_bytes=222019584 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=222128384 ingress_bytes=2335948800 link_bytes=2335948800 token_bytes=444147968 weight_movement_bytes=2113929216 total_nvlink_bytes=2558077184 weight_to_token_ratio=4.759516 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.308033821e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=226590720 dispatch_meta_bytes=110640 combine_payload_bytes=226590720 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=226701360 ingress_bytes=4454449152 link_bytes=4454449152 token_bytes=453292080 weight_movement_bytes=4227858432 total_nvlink_bytes=4681150512 weight_to_token_ratio=9.327007 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.494305587e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32532 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1924, 0, 8181, 4701, 67, 0, 1511] payload_rows_per_dst=[0, 3848, 0, 16260, 0, 236, 9402, 3022] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 3848, 0, 16260, 0, 236, 9402, 3022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134643712 dispatch_meta_bytes=65744 combine_payload_bytes=134643712 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134709456 ingress_bytes=1543929856 link_bytes=1543929856 token_bytes=269353168 weight_movement_bytes=1409286144 total_nvlink_bytes=1678639312 weight_to_token_ratio=5.232113 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645362726e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201375744 dispatch_meta_bytes=98328 combine_payload_bytes=201375744 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=201474072 ingress_bytes=5838520320 link_bytes=5838520320 token_bytes=402849816 weight_movement_bytes=5637144576 total_nvlink_bytes=6039994392 weight_to_token_ratio=13.993167 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269327667e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29176 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 567, 1, 8186, 5413, 18, 0, 2199] payload_rows_per_dst=[16372, 3592, 2, 162, 8232, 36, 0, 4372] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16372, 3592, 2, 162, 8232, 36, 0, 4372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=266502144 dispatch_meta_bytes=130128 combine_payload_bytes=266502144 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=266632272 ingress_bytes=1675788288 link_bytes=1675788288 token_bytes=533134416 weight_movement_bytes=1409286144 total_nvlink_bytes=1942420560 weight_to_token_ratio=2.643397 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.383714904e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24442 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 1, 8192, 0, 0, 8190, 0] payload_rows_per_dst=[2, 8058, 8192, 8326, 8190, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8058, 8192, 8326, 8190, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32750 payload_rows_remote=16406 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000549 router_tokens_per_expert=[3, 817, 5, 8169, 5802, 70, 15, 1503] payload_rows_per_dst=[16344, 13220, 10, 0, 1682, 140, 30, 1324] dup_rows_per_dst=[0, 18, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16344, 13220, 10, 0, 1682, 140, 30, 1324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=16386 dup_rows=12 dup_rows_remote=0 dedup_ratio=0.000366 router_tokens_per_expert=[8169, 0, 6, 13, 3, 41, 8135, 17] payload_rows_per_dst=[0, 0, 12, 26, 6, 8262, 8080, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 12] combine_rows_per_src=[0, 0, 12, 26, 6, 8262, 8080, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=32708 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[8189, 1, 0, 0, 6, 3, 8185, 0] payload_rows_per_dst=[0, 2, 8254, 8124, 58, 8146, 8182, 0] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[0, 2, 8254, 8124, 58, 8146, 8182, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=239009792 dispatch_meta_bytes=116704 combine_payload_bytes=239009792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=239126496 ingress_bytes=1648295936 link_bytes=1648295936 token_bytes=478136288 weight_movement_bytes=1409286144 total_nvlink_bytes=1887422432 weight_to_token_ratio=2.947457 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4992 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.229769209e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32750 payload_rows_remote=24634 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000549 router_tokens_per_expert=[8190, 9, 0, 1, 0, 15, 8169, 0] payload_rows_per_dst=[72, 8216, 8116, 34, 16282, 30, 0, 0] dup_rows_per_dst=[0, 18, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[72, 8216, 8116, 34, 16282, 30, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=24244 dup_rows=58 dup_rows_remote=58 dedup_ratio=0.001770 router_tokens_per_expert=[0, 1315, 0, 8192, 4233, 29, 0, 2615] payload_rows_per_dst=[0, 2630, 0, 18, 0, 16366, 8466, 5230] dup_rows_per_dst=[0, 0, 0, 0, 0, 58, 0, 0] combine_rows_per_src=[0, 2630, 0, 18, 0, 16366, 8466, 5230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=200228864 dispatch_meta_bytes=97768 combine_payload_bytes=200228864 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=200326632 ingress_bytes=5837373440 link_bytes=5837373440 token_bytes=400555496 weight_movement_bytes=5637144576 total_nvlink_bytes=6037700072 weight_to_token_ratio=14.073317 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.268685462e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134397952 dispatch_meta_bytes=65696 combine_payload_bytes=134397952 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134463648 ingress_bytes=1543684096 link_bytes=1543684096 token_bytes=268861600 weight_movement_bytes=1409286144 total_nvlink_bytes=1678147744 weight_to_token_ratio=5.241679 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643986572e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=267943936 dispatch_meta_bytes=130832 combine_payload_bytes=267943936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268074768 ingress_bytes=1677230080 link_bytes=1677230080 token_bytes=536018704 weight_movement_bytes=1409286144 total_nvlink_bytes=1945304848 weight_to_token_ratio=2.629173 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.391788338e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=24614 dup_rows=66 dup_rows_remote=0 dedup_ratio=0.002014 router_tokens_per_expert=[8155, 0, 0, 0, 1, 38, 8190, 0] payload_rows_per_dst=[0, 0, 8124, 8186, 2, 8088, 8302, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 66, 0, 0] combine_rows_per_src=[0, 0, 8124, 8186, 2, 8088, 8302, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=198606848 dispatch_meta_bytes=97208 combine_payload_bytes=198606848 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=198704056 ingress_bytes=1607892992 link_bytes=1607892992 token_bytes=397310904 weight_movement_bytes=1409286144 total_nvlink_bytes=1806597048 weight_to_token_ratio=3.547061 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.003529588e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201801728 dispatch_meta_bytes=98608 combine_payload_bytes=201801728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201900336 ingress_bytes=1611087872 link_bytes=1611087872 token_bytes=403702064 weight_movement_bytes=1409286144 total_nvlink_bytes=1812988208 weight_to_token_ratio=3.490906 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.021419582e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24554 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8175, 0, 0, 3, 0, 14, 8191, 1] payload_rows_per_dst=[16326, 8214, 0, 8198, 0, 28, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16326, 8214, 0, 8198, 0, 28, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201637888 dispatch_meta_bytes=98456 combine_payload_bytes=201637888 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201736344 ingress_bytes=1610924032 link_bytes=1610924032 token_bytes=403374232 weight_movement_bytes=1409286144 total_nvlink_bytes=1812660376 weight_to_token_ratio=3.493744 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.020502146e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1021, 0, 8192, 4804, 56, 0, 2311] payload_rows_per_dst=[0, 2042, 16384, 0, 9608, 112, 0, 4622] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2042, 16384, 0, 9608, 112, 0, 4622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32726 payload_rows_remote=16394 dup_rows=42 dup_rows_remote=4 dedup_ratio=0.001282 router_tokens_per_expert=[8166, 10, 8, 2, 22, 1, 8167, 8] payload_rows_per_dst=[16332, 8176, 16, 8140, 44, 2, 0, 16] dup_rows_per_dst=[38, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16332, 8176, 16, 8140, 44, 2, 0, 16] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=21708 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 2676, 8165, 1, 6, 3, 5530, 0] payload_rows_per_dst=[6, 0, 0, 852, 12, 16336, 4502, 11060] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6, 0, 0, 852, 12, 16336, 4502, 11060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=19892 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 4580, 8192, 2, 2, 0, 3608, 0] payload_rows_per_dst=[0, 0, 0, 12676, 12872, 0, 1040, 6176] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 12676, 12872, 0, 1040, 6176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201146368 dispatch_meta_bytes=98216 combine_payload_bytes=201146368 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201244584 ingress_bytes=1610432512 link_bytes=1610432512 token_bytes=402390952 weight_movement_bytes=1409286144 total_nvlink_bytes=1811677096 weight_to_token_ratio=3.502281 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.017749840e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 2, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=704643072 reduce_bytes=5637144576 egress_bytes=268566528 ingress_bytes=6610223104 link_bytes=6610223104 token_bytes=537001984 weight_movement_bytes=6341787648 total_nvlink_bytes=6878789632 weight_to_token_ratio=11.809617 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.701449014e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 6, 58, 0, 8134, 8204, 16366] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6, 58, 0, 8134, 8204, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16422 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 4758, 8192, 0, 1, 0, 3433, 0] payload_rows_per_dst=[38, 9516, 16346, 0, 2, 0, 6866, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[38, 9516, 16346, 0, 2, 0, 6866, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65592 combine_payload_bytes=134299648 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134365240 ingress_bytes=5066801152 link_bytes=5066801152 token_bytes=268664888 weight_movement_bytes=4932501504 total_nvlink_bytes=5201166392 weight_to_token_ratio=18.359308 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837197146e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 5338, 8188, 0, 0, 1, 2857, 0] payload_rows_per_dst=[0, 0, 0, 12502, 14506, 46, 0, 5714] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 12502, 14506, 46, 0, 5714] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 2, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=177831936 dispatch_meta_bytes=86832 combine_payload_bytes=177831936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=177918768 ingress_bytes=1587118080 link_bytes=1587118080 token_bytes=355750704 weight_movement_bytes=1409286144 total_nvlink_bytes=1765036848 weight_to_token_ratio=3.961443 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.887198752e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=162955264 dispatch_meta_bytes=79568 combine_payload_bytes=162955264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=163034832 ingress_bytes=1572241408 link_bytes=1572241408 token_bytes=325990096 weight_movement_bytes=1409286144 total_nvlink_bytes=1735276240 weight_to_token_ratio=4.323095 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.803895599e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201228288 dispatch_meta_bytes=98256 combine_payload_bytes=201228288 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201326544 ingress_bytes=5133729792 link_bytes=5133729792 token_bytes=402554832 weight_movement_bytes=4932501504 total_nvlink_bytes=5335056336 weight_to_token_ratio=12.252993 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874674391e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134529024 dispatch_meta_bytes=65688 combine_payload_bytes=134529024 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134594712 ingress_bytes=5771673600 link_bytes=5771673600 token_bytes=269123736 weight_movement_bytes=5637144576 total_nvlink_bytes=5906268312 weight_to_token_ratio=20.946293 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231896294e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=22590 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 5089, 8190, 1, 1, 1, 3081, 21] payload_rows_per_dst=[16380, 10178, 0, 2, 2, 2, 6162, 42] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 10178, 0, 2, 2, 2, 6162, 42] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268058624 dispatch_meta_bytes=130888 combine_payload_bytes=268058624 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268189512 ingress_bytes=1677344768 link_bytes=1677344768 token_bytes=536248136 weight_movement_bytes=1409286144 total_nvlink_bytes=1945534280 weight_to_token_ratio=2.628049 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.392430543e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8188, 0, 1, 1, 0, 6, 8187, 1] payload_rows_per_dst=[0, 8138, 8240, 2, 16374, 12, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8138, 8240, 2, 16374, 12, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16426 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8, 3836, 8163, 3, 8, 0, 4339, 27] payload_rows_per_dst=[16342, 7672, 0, 6, 16, 0, 8678, 54] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16342, 7672, 0, 6, 16, 0, 8678, 54] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=185057280 dispatch_meta_bytes=90360 combine_payload_bytes=185057280 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=185147640 ingress_bytes=4412915712 link_bytes=4412915712 token_bytes=370204920 weight_movement_bytes=4227858432 total_nvlink_bytes=4598063352 weight_to_token_ratio=11.420319 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.471048595e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32724 payload_rows_remote=16422 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.001343 router_tokens_per_expert=[22, 10, 8107, 8164, 0, 6, 31, 44] payload_rows_per_dst=[44, 20, 0, 0, 0, 8204, 8154, 16302] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 44, 0] combine_rows_per_src=[44, 20, 0, 0, 0, 8204, 8154, 16302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16484 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 8176, 8178, 0, 1, 0, 28] payload_rows_per_dst=[0, 2, 244, 72, 16284, 8166, 7944, 56] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 244, 72, 16284, 8166, 7944, 56] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30222 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 4082, 8192, 0, 0, 0, 4109, 0] payload_rows_per_dst=[2, 0, 0, 4092, 1526, 16384, 2546, 8218] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 4092, 1526, 16384, 2546, 8218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32736 payload_rows_remote=16610 dup_rows=32 dup_rows_remote=0 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 8172, 8190, 0, 0, 10, 12] payload_rows_per_dst=[20, 230, 16126, 16316, 0, 0, 20, 24] dup_rows_per_dst=[0, 0, 32, 0, 0, 0, 0, 0] combine_rows_per_src=[20, 230, 16126, 16316, 0, 0, 20, 24] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134561792 dispatch_meta_bytes=65704 combine_payload_bytes=134561792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134627496 ingress_bytes=1543847936 link_bytes=1543847936 token_bytes=269189288 weight_movement_bytes=1409286144 total_nvlink_bytes=1678475432 weight_to_token_ratio=5.235298 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644904008e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24592 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8188, 8188, 0, 0, 3, 5] payload_rows_per_dst=[0, 0, 0, 0, 16376, 8176, 8192, 24] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16376, 8176, 8192, 24] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134529024 dispatch_meta_bytes=65864 combine_payload_bytes=134529024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134594888 ingress_bytes=1543815168 link_bytes=1543815168 token_bytes=269123912 weight_movement_bytes=1409286144 total_nvlink_bytes=1678410056 weight_to_token_ratio=5.236570 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644720521e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=135036928 dispatch_meta_bytes=65936 combine_payload_bytes=135036928 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135102864 ingress_bytes=1544323072 link_bytes=1544323072 token_bytes=270139792 weight_movement_bytes=1409286144 total_nvlink_bytes=1679425936 weight_to_token_ratio=5.216877 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.647564571e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=247578624 dispatch_meta_bytes=120888 combine_payload_bytes=247578624 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=247699512 ingress_bytes=3066150912 link_bytes=3066150912 token_bytes=495278136 weight_movement_bytes=2818572288 total_nvlink_bytes=3313850424 weight_to_token_ratio=5.690888 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7424 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.716916523e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=136069120 dispatch_meta_bytes=66440 combine_payload_bytes=136069120 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=136135560 ingress_bytes=5773213696 link_bytes=5773213696 token_bytes=272204680 weight_movement_bytes=5637144576 total_nvlink_bytes=5909349256 weight_to_token_ratio=20.709213 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.232758684e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=16418 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 8181, 8192, 0, 0, 11, 0] payload_rows_per_dst=[16378, 16344, 18, 0, 0, 0, 22, 0] dup_rows_per_dst=[0, 0, 6, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 16344, 18, 0, 0, 0, 22, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=30066 dup_rows=64 dup_rows_remote=64 dedup_ratio=0.001953 router_tokens_per_expert=[0, 5954, 8189, 0, 2, 0, 2239, 0] payload_rows_per_dst=[0, 5402, 16358, 2638, 3828, 0, 2974, 1504] dup_rows_per_dst=[0, 0, 64, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 5402, 16358, 2638, 3828, 0, 2974, 1504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=16444 dup_rows=8 dup_rows_remote=2 dedup_ratio=0.000244 router_tokens_per_expert=[21, 15, 8052, 8158, 2, 11, 24, 101] payload_rows_per_dst=[16316, 16168, 0, 0, 4, 22, 48, 202] dup_rows_per_dst=[6, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16316, 16168, 0, 0, 4, 22, 48, 202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32740 payload_rows_remote=16426 dup_rows=28 dup_rows_remote=0 dedup_ratio=0.000854 router_tokens_per_expert=[5853, 2305, 18, 21, 8155, 3, 13, 16] payload_rows_per_dst=[0, 4610, 36, 42, 0, 7178, 4560, 16314] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 28] combine_rows_per_src=[0, 4610, 36, 42, 0, 7178, 4560, 16314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=201457664 dispatch_meta_bytes=98368 combine_payload_bytes=201457664 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201556032 ingress_bytes=1610743808 link_bytes=1610743808 token_bytes=403013696 weight_movement_bytes=1409286144 total_nvlink_bytes=1812299840 weight_to_token_ratio=3.496869 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019492967e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134496256 dispatch_meta_bytes=65696 combine_payload_bytes=134496256 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134561952 ingress_bytes=1543782400 link_bytes=1543782400 token_bytes=269058208 weight_movement_bytes=1409286144 total_nvlink_bytes=1678344352 weight_to_token_ratio=5.237849 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644537034e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=246300672 dispatch_meta_bytes=120520 combine_payload_bytes=246300672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=246421192 ingress_bytes=1655586816 link_bytes=1655586816 token_bytes=492721864 weight_movement_bytes=1409286144 total_nvlink_bytes=1902008008 weight_to_token_ratio=2.860206 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.270595094e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134709248 dispatch_meta_bytes=65784 combine_payload_bytes=134709248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134775032 ingress_bytes=1543995392 link_bytes=1543995392 token_bytes=269484280 weight_movement_bytes=1409286144 total_nvlink_bytes=1678770424 weight_to_token_ratio=5.229567 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645729700e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134561792 dispatch_meta_bytes=65704 combine_payload_bytes=134561792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134627496 ingress_bytes=1543847936 link_bytes=1543847936 token_bytes=269189288 weight_movement_bytes=1409286144 total_nvlink_bytes=1678475432 weight_to_token_ratio=5.235298 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644904008e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32748 payload_rows_remote=32670 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000610 router_tokens_per_expert=[6976, 1189, 3, 10, 8190, 6, 10, 0] payload_rows_per_dst=[0, 2378, 6846, 7132, 78, 8126, 8188, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 20, 0] combine_rows_per_src=[0, 2378, 6846, 7132, 78, 8126, 8188, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=31410 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5728, 2447, 0, 17, 8192, 0, 0, 0] payload_rows_per_dst=[8126, 6944, 1358, 34, 16306, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8126, 6944, 1358, 34, 16306, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24428 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8190, 8192, 0, 0, 2, 0] payload_rows_per_dst=[0, 0, 0, 0, 104, 8170, 8340, 16154] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 104, 8170, 8340, 16154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19960 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7882, 306, 0, 4, 8191, 0, 1, 0] payload_rows_per_dst=[0, 612, 3040, 2640, 0, 12808, 13604, 64] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 612, 3040, 2640, 0, 12808, 13604, 64] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=267632640 dispatch_meta_bytes=130760 combine_payload_bytes=267632640 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=267763400 ingress_bytes=5904777216 link_bytes=5904777216 token_bytes=535396040 weight_movement_bytes=5637144576 total_nvlink_bytes=6172540616 weight_to_token_ratio=10.528925 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306428763e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27656 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5621, 2556, 1, 15, 8190, 0, 1, 0] payload_rows_per_dst=[11242, 5112, 8190, 8202, 20, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[11242, 5112, 8190, 8202, 20, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=257310720 dispatch_meta_bytes=125640 combine_payload_bytes=257310720 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=257436360 ingress_bytes=1666596864 link_bytes=1666596864 token_bytes=514747080 weight_movement_bytes=1409286144 total_nvlink_bytes=1924033224 weight_to_token_ratio=2.737823 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.332246767e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32754 payload_rows_remote=19948 dup_rows=14 dup_rows_remote=0 dedup_ratio=0.000427 router_tokens_per_expert=[5653, 2519, 22, 2, 8146, 23, 19, 0] payload_rows_per_dst=[12806, 3580, 8128, 8156, 0, 46, 38, 0] dup_rows_per_dst=[14, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12806, 3580, 8128, 8156, 0, 46, 38, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16440 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8, 3, 985, 7196, 0, 24, 4, 8164] payload_rows_per_dst=[16, 6, 1970, 0, 7346, 48, 7054, 16328] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16, 6, 1970, 0, 7346, 48, 7054, 16328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=200114176 dispatch_meta_bytes=97712 combine_payload_bytes=200114176 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=200211888 ingress_bytes=1609400320 link_bytes=1609400320 token_bytes=400326064 weight_movement_bytes=1409286144 total_nvlink_bytes=1809612208 weight_to_token_ratio=3.520346 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.011969995e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=163512320 dispatch_meta_bytes=79840 combine_payload_bytes=163512320 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=163592160 ingress_bytes=1572798464 link_bytes=1572798464 token_bytes=327104480 weight_movement_bytes=1409286144 total_nvlink_bytes=1736390624 weight_to_token_ratio=4.308367 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.807014880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 1, 8183, 8190, 0, 3, 2, 3] payload_rows_per_dst=[4, 2, 16342, 16380, 0, 30, 4, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 2, 16342, 16380, 0, 30, 4, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 0, 2, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=226557952 dispatch_meta_bytes=110624 combine_payload_bytes=226557952 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=226668576 ingress_bytes=2340487168 link_bytes=2340487168 token_bytes=453226528 weight_movement_bytes=2113929216 total_nvlink_bytes=2567155744 weight_to_token_ratio=4.664178 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.310575117e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134676480 dispatch_meta_bytes=65760 combine_payload_bytes=134676480 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134742240 ingress_bytes=5067177984 link_bytes=5067177984 token_bytes=269418720 weight_movement_bytes=4932501504 total_nvlink_bytes=5201920224 weight_to_token_ratio=18.307939 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837408157e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32752 payload_rows_remote=24566 dup_rows=16 dup_rows_remote=0 dedup_ratio=0.000488 router_tokens_per_expert=[3, 0, 1140, 7039, 10, 0, 0, 8192] payload_rows_per_dst=[6, 0, 2280, 6890, 8186, 7208, 8182, 0] dup_rows_per_dst=[0, 0, 0, 0, 16, 0, 0, 0] combine_rows_per_src=[6, 0, 2280, 6890, 8186, 7208, 8182, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=163414016 dispatch_meta_bytes=79792 combine_payload_bytes=163414016 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=163493808 ingress_bytes=5095915520 link_bytes=5095915520 token_bytes=326907824 weight_movement_bytes=4932501504 total_nvlink_bytes=5259409328 weight_to_token_ratio=15.088356 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.853499977e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134316048 ingress_bytes=5771395072 link_bytes=5771395072 token_bytes=268566544 weight_movement_bytes=5637144576 total_nvlink_bytes=5905711120 weight_to_token_ratio=20.989750 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231740330e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1747, 6442, 1, 2, 0, 8192] payload_rows_per_dst=[50, 9362, 6116, 9092, 2, 8146, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[50, 9362, 6116, 9092, 2, 8146, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=30346 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 3667, 4523, 2, 0, 0, 8192] payload_rows_per_dst=[0, 0, 7334, 1340, 11218, 2418, 10398, 56] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[0, 0, 7334, 1340, 11218, 2418, 10398, 56] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26428 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5460, 2731, 0, 1, 8191, 0, 0, 1] payload_rows_per_dst=[0, 5462, 0, 2, 0, 4608, 6340, 16356] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 5462, 0, 2, 0, 4608, 6340, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201244672 dispatch_meta_bytes=98264 combine_payload_bytes=201244672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201342936 ingress_bytes=1610530816 link_bytes=1610530816 token_bytes=402587608 weight_movement_bytes=1409286144 total_nvlink_bytes=1811873752 weight_to_token_ratio=3.500570 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018300301e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25012 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 1103, 7082, 5, 0, 1, 8191] payload_rows_per_dst=[16374, 7756, 2206, 6420, 10, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16374, 7756, 2206, 6420, 10, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=218333184 dispatch_meta_bytes=106608 combine_payload_bytes=218333184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=218439792 ingress_bytes=1627619328 link_bytes=1627619328 token_bytes=436772976 weight_movement_bytes=1409286144 total_nvlink_bytes=1846059120 weight_to_token_ratio=3.226587 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.113988835e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32752 payload_rows_remote=16426 dup_rows=16 dup_rows_remote=0 dedup_ratio=0.000488 router_tokens_per_expert=[0, 7478, 1, 23, 702, 2, 15, 8163] payload_rows_per_dst=[0, 0, 2, 46, 1404, 7584, 7390, 16326] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 16] combine_rows_per_src=[0, 0, 2, 46, 1404, 7584, 7390, 16326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=248594432 dispatch_meta_bytes=121400 combine_payload_bytes=248594432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=248715832 ingress_bytes=1657880576 link_bytes=1657880576 token_bytes=497310264 weight_movement_bytes=1409286144 total_nvlink_bytes=1906596408 weight_to_token_ratio=2.833817 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.283439192e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16446 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 14, 1033, 7145, 5, 21, 5, 8159] payload_rows_per_dst=[16322, 7462, 7328, 1594, 10, 42, 10, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16322, 7462, 7328, 1594, 10, 42, 10, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=216498176 dispatch_meta_bytes=105712 combine_payload_bytes=216498176 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=216603888 ingress_bytes=1625784320 link_bytes=1625784320 token_bytes=433102064 weight_movement_bytes=1409286144 total_nvlink_bytes=1842388208 weight_to_token_ratio=3.253935 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.103713556e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26208 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[6153, 2035, 2, 0, 8190, 1, 0, 3] payload_rows_per_dst=[580, 4070, 5170, 6560, 16364, 2, 16, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[580, 4070, 5170, 6560, 16364, 2, 16, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=204898304 dispatch_meta_bytes=100048 combine_payload_bytes=204898304 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=204998352 ingress_bytes=1614184448 link_bytes=1614184448 token_bytes=409896656 weight_movement_bytes=1409286144 total_nvlink_bytes=1819182800 weight_to_token_ratio=3.438150 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.038759115e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134561792 dispatch_meta_bytes=65704 combine_payload_bytes=134561792 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=134627496 ingress_bytes=5771706368 link_bytes=5771706368 token_bytes=269189288 weight_movement_bytes=5637144576 total_nvlink_bytes=5906333864 weight_to_token_ratio=20.941192 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231914643e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26234 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7792, 0, 2, 402, 2, 1, 8185] payload_rows_per_dst=[0, 2082, 16, 8192, 6534, 7776, 8168, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2082, 16, 8192, 6534, 7776, 8168, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134725632 dispatch_meta_bytes=65784 combine_payload_bytes=134725632 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134791416 ingress_bytes=1544011776 link_bytes=1544011776 token_bytes=269517048 weight_movement_bytes=1409286144 total_nvlink_bytes=1678803192 weight_to_token_ratio=5.228931 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645821443e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24518 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7767, 0, 0, 425, 0, 0, 8192] payload_rows_per_dst=[1316, 7408, 8250, 8118, 7676, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[1316, 7408, 8250, 8118, 7676, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=214695936 dispatch_meta_bytes=104832 combine_payload_bytes=214695936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=214800768 ingress_bytes=1623982080 link_bytes=1623982080 token_bytes=429496704 weight_movement_bytes=1409286144 total_nvlink_bytes=1838782848 weight_to_token_ratio=3.281250 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.093621765e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24952 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7795, 0, 0, 397, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 8154, 7356, 7816, 9400, 42] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8154, 7356, 7816, 9400, 42] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=214908928 dispatch_meta_bytes=104936 combine_payload_bytes=214908928 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=215013864 ingress_bytes=1624195072 link_bytes=1624195072 token_bytes=429922792 weight_movement_bytes=1409286144 total_nvlink_bytes=1839208936 weight_to_token_ratio=3.277998 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.094814431e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1647, 6534, 11, 0, 0, 8192] payload_rows_per_dst=[0, 0, 3294, 0, 5972, 0, 7118, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 3294, 0, 5972, 0, 7118, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24994 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[12, 7744, 0, 2, 436, 0, 0, 8190] payload_rows_per_dst=[15930, 7774, 8188, 4, 872, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15930, 7774, 8188, 4, 872, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=200851456 dispatch_meta_bytes=98072 combine_payload_bytes=200851456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=200949528 ingress_bytes=1610137600 link_bytes=1610137600 token_bytes=401800984 weight_movement_bytes=1409286144 total_nvlink_bytes=1811087128 weight_to_token_ratio=3.507423 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.016098455e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=204406784 dispatch_meta_bytes=99808 combine_payload_bytes=204406784 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=204506592 ingress_bytes=1613692928 link_bytes=1613692928 token_bytes=408913376 weight_movement_bytes=1409286144 total_nvlink_bytes=1818199520 weight_to_token_ratio=3.446417 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.036006808e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 2, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32740 payload_rows_remote=16416 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000854 router_tokens_per_expert=[12, 8165, 8140, 0, 23, 19, 3, 22] payload_rows_per_dst=[24, 0, 0, 0, 8160, 8226, 6, 16324] dup_rows_per_dst=[0, 0, 0, 0, 28, 0, 0, 0] combine_rows_per_src=[24, 0, 0, 0, 8160, 8226, 6, 16324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17248 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4, 7497, 0, 29, 672, 3, 16, 8163] payload_rows_per_dst=[15520, 7686, 8122, 58, 1344, 6, 32, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15520, 7686, 8122, 58, 1344, 6, 32, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=210124800 dispatch_meta_bytes=102600 combine_payload_bytes=210124800 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=210227400 ingress_bytes=1619410944 link_bytes=1619410944 token_bytes=420352200 weight_movement_bytes=1409286144 total_nvlink_bytes=1829638344 weight_to_token_ratio=3.352632 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.068025311e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=204750848 dispatch_meta_bytes=99976 combine_payload_bytes=204750848 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=204850824 ingress_bytes=5841895424 link_bytes=5841895424 token_bytes=409601672 weight_movement_bytes=5637144576 total_nvlink_bytes=6046746248 weight_to_token_ratio=13.762504 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.271217585e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25336 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1120, 7070, 2, 0, 2, 8190] payload_rows_per_dst=[0, 8182, 2240, 7432, 4, 14906, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8182, 2240, 7432, 4, 14906, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 8185, 0, 0, 0, 8, 0] payload_rows_per_dst=[0, 2, 34, 16346, 116, 64, 16206, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 34, 16346, 116, 64, 16206, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134479872 dispatch_meta_bytes=65776 combine_payload_bytes=134479872 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134545648 ingress_bytes=1543766016 link_bytes=1543766016 token_bytes=269025520 weight_movement_bytes=1409286144 total_nvlink_bytes=1678311664 weight_to_token_ratio=5.238485 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644445290e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 2, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=141295616 dispatch_meta_bytes=68992 combine_payload_bytes=141295616 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=141364608 ingress_bytes=1550581760 link_bytes=1550581760 token_bytes=282660224 weight_movement_bytes=1409286144 total_nvlink_bytes=1691946368 weight_to_token_ratio=4.985796 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.682610611e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32520 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[2, 16382, 248, 0, 8040, 8096, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 16382, 248, 0, 8040, 8096, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=207552512 dispatch_meta_bytes=101344 combine_payload_bytes=207552512 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=207653856 ingress_bytes=5140054016 link_bytes=5140054016 token_bytes=415206368 weight_movement_bytes=4932501504 total_nvlink_bytes=5347707872 weight_to_token_ratio=11.879638 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.878215693e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8191, 0, 0, 0, 0, 1] payload_rows_per_dst=[0, 0, 0, 16384, 0, 0, 16382, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 0, 16382, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=267485184 dispatch_meta_bytes=130608 combine_payload_bytes=267485184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267615792 ingress_bytes=1676771328 link_bytes=1676771328 token_bytes=535100976 weight_movement_bytes=1409286144 total_nvlink_bytes=1944387120 weight_to_token_ratio=2.633683 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.389219518e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24990 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 7687, 0, 0, 504, 0, 0, 8192] payload_rows_per_dst=[2, 0, 0, 0, 1008, 7596, 7778, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 0, 1008, 7596, 7778, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 8189, 8174, 0, 17, 1, 0, 0] payload_rows_per_dst=[16372, 12, 16348, 0, 34, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16372, 12, 16348, 0, 34, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=266403840 dispatch_meta_bytes=130080 combine_payload_bytes=266403840 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=266533920 ingress_bytes=5903548416 link_bytes=5903548416 token_bytes=532937760 weight_movement_bytes=5637144576 total_nvlink_bytes=6170082336 weight_to_token_ratio=10.577491 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.305740687e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=204718080 dispatch_meta_bytes=99960 combine_payload_bytes=204718080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=204818040 ingress_bytes=1614004224 link_bytes=1614004224 token_bytes=409536120 weight_movement_bytes=1409286144 total_nvlink_bytes=1818822264 weight_to_token_ratio=3.441177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.037749936e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32758 payload_rows_remote=16388 dup_rows=10 dup_rows_remote=0 dedup_ratio=0.000305 router_tokens_per_expert=[8, 8160, 7510, 2, 0, 21, 653, 30] payload_rows_per_dst=[16, 0, 0, 4, 5450, 3408, 7510, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 10] combine_rows_per_src=[16, 0, 0, 4, 5450, 3408, 7510, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16400 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[25, 8159, 8067, 3, 6, 8, 0, 116] payload_rows_per_dst=[16368, 0, 16134, 6, 12, 16, 0, 232] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16368, 0, 16134, 6, 12, 16, 0, 232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268337152 dispatch_meta_bytes=131024 combine_payload_bytes=268337152 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=268468176 ingress_bytes=5905481728 link_bytes=5905481728 token_bytes=536805328 weight_movement_bytes=5637144576 total_nvlink_bytes=6173949904 weight_to_token_ratio=10.501283 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306823261e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24570 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 7743, 4, 2, 448, 0, 0, 8187] payload_rows_per_dst=[0, 7818, 8190, 8196, 6572, 1990, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[0, 7818, 8190, 8196, 6572, 1990, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24598 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8173, 7137, 0, 0, 2, 1069, 3] payload_rows_per_dst=[0, 0, 13466, 18, 8170, 8970, 2138, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 13466, 18, 8170, 8970, 2138, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134348800 dispatch_meta_bytes=65600 combine_payload_bytes=134348800 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134414400 ingress_bytes=1543634944 link_bytes=1543634944 token_bytes=268763200 weight_movement_bytes=1409286144 total_nvlink_bytes=1678049344 weight_to_token_ratio=5.243598 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643711341e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30132 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8166, 5729, 0, 0, 0, 2489, 0] payload_rows_per_dst=[8718, 120, 2636, 16316, 0, 0, 4978, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8718, 120, 2636, 16316, 0, 0, 4978, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201277440 dispatch_meta_bytes=98288 combine_payload_bytes=201277440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201375728 ingress_bytes=1610563584 link_bytes=1610563584 token_bytes=402653168 weight_movement_bytes=1409286144 total_nvlink_bytes=1811939312 weight_to_token_ratio=3.500000 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018483788e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18666 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8190, 6290, 0, 0, 0, 1904, 0] payload_rows_per_dst=[0, 0, 2248, 0, 12602, 14102, 3808, 8] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2248, 0, 12602, 14102, 3808, 8] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201506816 dispatch_meta_bytes=98392 combine_payload_bytes=201506816 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201605208 ingress_bytes=1610792960 link_bytes=1610792960 token_bytes=403112024 weight_movement_bytes=1409286144 total_nvlink_bytes=1812398168 weight_to_token_ratio=3.496016 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019768198e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8187, 0, 0, 0, 0, 5] payload_rows_per_dst=[0, 0, 0, 32, 8176, 8176, 174, 16210] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 32, 8176, 8176, 174, 16210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4539116696 prefetch_bytes=18320719872 reduce_bytes=33822867456 weight_movement_bytes=52143587328 egress_bytes=2270112920 ingress_bytes=54412591104 link_bytes=54412591104 total_nvlink_bytes=56682704024 weight_to_token_ratio=11.487607 prefetch_share_of_ingress=0.336700 dedup_ratio=0.007294 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=48 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8001 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.904298733e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16416 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8190, 5731, 0, 1, 2, 2459, 0] payload_rows_per_dst=[11464, 16352, 0, 28, 2, 4, 4918, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[11464, 16352, 0, 28, 2, 4, 4918, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=246841344 dispatch_meta_bytes=120528 combine_payload_bytes=246841344 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=246961872 ingress_bytes=5883985920 link_bytes=5883985920 token_bytes=493803216 weight_movement_bytes=5637144576 total_nvlink_bytes=6130947792 weight_to_token_ratio=11.415771 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.294786506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=152911872 dispatch_meta_bytes=74664 combine_payload_bytes=152911872 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=152986536 ingress_bytes=1562198016 link_bytes=1562198016 token_bytes=305898408 weight_movement_bytes=1409286144 total_nvlink_bytes=1715184552 weight_to_token_ratio=4.607040 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.747656796e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6653606048 prefetch_bytes=20434649088 reduce_bytes=11978932224 weight_movement_bytes=32413581312 egress_bytes=3327621280 ingress_bytes=35739566080 link_bytes=35739566080 total_nvlink_bytes=39067187360 weight_to_token_ratio=4.871581 prefetch_share_of_ingress=0.571765 dedup_ratio=0.006008 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=17 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8001 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.250791573e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=267010048 dispatch_meta_bytes=130376 combine_payload_bytes=267010048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267140424 ingress_bytes=1676296192 link_bytes=1676296192 token_bytes=534150472 weight_movement_bytes=1409286144 total_nvlink_bytes=1943436616 weight_to_token_ratio=2.638369 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.386558955e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=20200 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8, 8148, 6276, 14, 15, 24, 1889, 10] payload_rows_per_dst=[12568, 16296, 0, 28, 30, 48, 3778, 20] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12568, 16296, 0, 28, 30, 48, 3778, 20] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134479872 dispatch_meta_bytes=65664 combine_payload_bytes=134479872 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134545536 ingress_bytes=5771624448 link_bytes=5771624448 token_bytes=269025408 weight_movement_bytes=5637144576 total_nvlink_bytes=5906169984 weight_to_token_ratio=20.953949 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231868771e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8188, 8189, 0, 5, 1, 0, 0] payload_rows_per_dst=[2, 16372, 4, 0, 8202, 8188, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 16372, 4, 0, 8202, 8188, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6810952048 prefetch_bytes=21139292160 reduce_bytes=21139292160 weight_movement_bytes=42278584320 egress_bytes=3406307696 ingress_bytes=45683228672 link_bytes=45683228672 total_nvlink_bytes=49089536368 weight_to_token_ratio=6.207441 prefetch_share_of_ingress=0.462736 dedup_ratio=0.000763 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=30 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8001 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.598793822e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6865504992 prefetch_bytes=21139292160 reduce_bytes=4227858432 weight_movement_bytes=25367150592 egress_bytes=3433597664 ingress_bytes=28799057920 link_bytes=28799057920 total_nvlink_bytes=32232655584 weight_to_token_ratio=3.694870 prefetch_share_of_ingress=0.734027 dedup_ratio=0.009357 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=6 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016098 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.007891894e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=165478400 dispatch_meta_bytes=80800 combine_payload_bytes=165478400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=165559200 ingress_bytes=1574764544 link_bytes=1574764544 token_bytes=331037600 weight_movement_bytes=1409286144 total_nvlink_bytes=1740323744 weight_to_token_ratio=4.257178 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.818024107e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6710067904 prefetch_bytes=19025362944 reduce_bytes=21139292160 weight_movement_bytes=40164655104 egress_bytes=3355853504 ingress_bytes=43518869504 link_bytes=43518869504 total_nvlink_bytes=46874723008 weight_to_token_ratio=5.985730 prefetch_share_of_ingress=0.437175 dedup_ratio=0.000648 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=30 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.523046897e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29096 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8185, 6363, 0, 0, 0, 1836, 0] payload_rows_per_dst=[0, 0, 0, 0, 6508, 6234, 3672, 16354] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 6508, 6234, 3672, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4621319568 prefetch_bytes=19025362944 reduce_bytes=21843935232 weight_movement_bytes=40869298176 egress_bytes=2311224720 ingress_bytes=43179393024 link_bytes=43179393024 total_nvlink_bytes=45490617744 weight_to_token_ratio=8.843643 prefetch_share_of_ingress=0.440612 dedup_ratio=0.009296 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=31 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.511166106e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=238354432 dispatch_meta_bytes=116384 combine_payload_bytes=238354432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=238470816 ingress_bytes=1647640576 link_bytes=1647640576 token_bytes=476825248 weight_movement_bytes=1409286144 total_nvlink_bytes=1886111392 weight_to_token_ratio=2.955561 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7168 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.226099467e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8187, 7209, 0, 3, 1, 983, 0] payload_rows_per_dst=[2, 0, 14418, 16374, 6, 2, 1966, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 14418, 16374, 6, 2, 1966, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6956878944 prefetch_bytes=20434649088 reduce_bytes=15502147584 weight_movement_bytes=35936796672 egress_bytes=3479293024 ingress_bytes=39414382592 link_bytes=39414382592 total_nvlink_bytes=42893675616 weight_to_token_ratio=5.165649 prefetch_share_of_ingress=0.518457 dedup_ratio=0.004459 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=22 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.379400563e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134365224 ingress_bytes=1543585792 link_bytes=1543585792 token_bytes=268664872 weight_movement_bytes=1409286144 total_nvlink_bytes=1677951016 weight_to_token_ratio=5.245517 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643436111e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6747530072 prefetch_bytes=16911433728 reduce_bytes=26776436736 weight_movement_bytes=43687870464 egress_bytes=3374588760 ingress_bytes=47060811776 link_bytes=47060811776 total_nvlink_bytes=50435400536 weight_to_token_ratio=6.474646 prefetch_share_of_ingress=0.359353 dedup_ratio=0.000259 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=38 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.647005636e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none - [2026-09-16 21:34:41] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 611.3 | throughput per GPU (TFLOP/s/GPU): 545.4 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.119646E+01 | loss scale: 1.0 | grad norm: 89.114 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32730 payload_rows_remote=25882 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.001160 router_tokens_per_expert=[32, 6, 85, 3699, 2337, 3820, 3005, 3400] payload_rows_per_dst=[6872, 6848, 6938, 7398, 4674, 0, 0, 0] dup_rows_per_dst=[6, 0, 32, 0, 0, 0, 0, 0] combine_rows_per_src=[6872, 6848, 6938, 7398, 4674, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32362 payload_rows_remote=23822 dup_rows=406 dup_rows_remote=406 dedup_ratio=0.012390 router_tokens_per_expert=[17, 6, 106, 2503, 2190, 4475, 4207, 2880] payload_rows_per_dst=[442, 12, 212, 1734, 6888, 8540, 8414, 6120] dup_rows_per_dst=[2, 0, 0, 0, 380, 0, 0, 24] combine_rows_per_src=[442, 12, 212, 1734, 6888, 8540, 8414, 6120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=23858 dup_rows=60 dup_rows_remote=12 dedup_ratio=0.001831 router_tokens_per_expert=[37, 3, 166, 2743, 2016, 2987, 4149, 4283] payload_rows_per_dst=[6456, 7884, 8850, 5486, 4032, 0, 0, 0] dup_rows_per_dst=[12, 0, 48, 0, 0, 0, 0, 0] combine_rows_per_src=[6456, 7884, 8850, 5486, 4032, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32648 payload_rows_remote=26688 dup_rows=120 dup_rows_remote=120 dedup_ratio=0.003662 router_tokens_per_expert=[11, 0, 99, 2698, 2980, 3376, 4206, 3014] payload_rows_per_dst=[3444, 1594, 2240, 5710, 5960, 2852, 6872, 3976] dup_rows_per_dst=[2, 0, 10, 108, 0, 0, 0, 0] combine_rows_per_src=[3444, 1594, 2240, 5710, 5960, 2852, 6872, 3976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30952 payload_rows_remote=19510 dup_rows=1816 dup_rows_remote=830 dedup_ratio=0.055420 router_tokens_per_expert=[39, 12, 142, 2283, 1601, 4740, 3352, 4215] payload_rows_per_dst=[78, 24, 284, 0, 1088, 11332, 6704, 11442] dup_rows_per_dst=[0, 0, 0, 0, 0, 830, 0, 986] combine_rows_per_src=[78, 24, 284, 0, 1088, 11332, 6704, 11442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30880 payload_rows_remote=20412 dup_rows=1888 dup_rows_remote=1578 dedup_ratio=0.057617 router_tokens_per_expert=[9, 3, 60, 2882, 1855, 3520, 4817, 3238] payload_rows_per_dst=[18, 6, 120, 0, 2394, 8422, 10468, 9452] dup_rows_per_dst=[0, 0, 0, 0, 18, 792, 310, 768] combine_rows_per_src=[18, 6, 120, 0, 2394, 8422, 10468, 9452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32722 payload_rows_remote=26760 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.001404 router_tokens_per_expert=[28, 3, 127, 2981, 1767, 2857, 4599, 4022] payload_rows_per_dst=[7560, 7400, 8266, 5962, 3534, 0, 0, 0] dup_rows_per_dst=[14, 0, 32, 0, 0, 0, 0, 0] combine_rows_per_src=[7560, 7400, 8266, 5962, 3534, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=24868 dup_rows=48 dup_rows_remote=38 dedup_ratio=0.001465 router_tokens_per_expert=[45, 0, 131, 3185, 1900, 4544, 3842, 2737] payload_rows_per_dst=[7852, 9000, 5698, 6370, 3800, 0, 0, 0] dup_rows_per_dst=[10, 0, 38, 0, 0, 0, 0, 0] combine_rows_per_src=[7852, 9000, 5698, 6370, 3800, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=212025344 dispatch_meta_bytes=103680 combine_payload_bytes=212025344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=212129024 ingress_bytes=1621311488 link_bytes=1621311488 token_bytes=424154368 weight_movement_bytes=1409286144 total_nvlink_bytes=1833440512 weight_to_token_ratio=3.322578 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.078667564e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=195149824 dispatch_meta_bytes=96912 combine_payload_bytes=195149824 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=195246736 ingress_bytes=3013722112 link_bytes=3013722112 token_bytes=390396560 weight_movement_bytes=2818572288 total_nvlink_bytes=3208968848 weight_to_token_ratio=7.219767 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.687558584e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=195444736 dispatch_meta_bytes=95480 combine_payload_bytes=195444736 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=195540216 ingress_bytes=1604730880 link_bytes=1604730880 token_bytes=390984952 weight_movement_bytes=1409286144 total_nvlink_bytes=1800271096 weight_to_token_ratio=3.604451 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.985823080e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=218628096 dispatch_meta_bytes=107232 combine_payload_bytes=218628096 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=218735328 ingress_bytes=2332557312 link_bytes=2332557312 token_bytes=437363424 weight_movement_bytes=2113929216 total_nvlink_bytes=2551292640 weight_to_token_ratio=4.833347 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7296 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.306134729e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=159825920 dispatch_meta_bytes=81360 combine_payload_bytes=159825920 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=159907280 ingress_bytes=2978398208 link_bytes=2978398208 token_bytes=319733200 weight_movement_bytes=2818572288 total_nvlink_bytes=3138305488 weight_to_token_ratio=8.815388 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6400 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.667778672e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=167215104 dispatch_meta_bytes=87960 combine_payload_bytes=167215104 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=167303064 ingress_bytes=2985787392 link_bytes=2985787392 token_bytes=334518168 weight_movement_bytes=2818572288 total_nvlink_bytes=3153090456 weight_to_token_ratio=8.425767 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.671916307e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=219217920 dispatch_meta_bytes=107224 combine_payload_bytes=219217920 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=219325144 ingress_bytes=5151719424 link_bytes=5151719424 token_bytes=438543064 weight_movement_bytes=4932501504 total_nvlink_bytes=5371044568 weight_to_token_ratio=11.247474 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.884747834e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=203718656 dispatch_meta_bytes=99624 combine_payload_bytes=203718656 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=203818280 ingress_bytes=1613004800 link_bytes=1613004800 token_bytes=407536936 weight_movement_bytes=1409286144 total_nvlink_bytes=1816823080 weight_to_token_ratio=3.458057 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.032153579e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32658 payload_rows_remote=29790 dup_rows=110 dup_rows_remote=110 dedup_ratio=0.003357 router_tokens_per_expert=[7227, 2, 0, 2, 5290, 2682, 331, 850] payload_rows_per_dst=[0, 4, 7196, 7262, 8130, 2868, 3032, 4166] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 80, 30] combine_rows_per_src=[0, 4, 7196, 7262, 8130, 2868, 3032, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29154 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[6692, 1, 3, 0, 6053, 2341, 440, 854] payload_rows_per_dst=[3636, 12356, 3614, 5892, 0, 4682, 880, 1708] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3636, 12356, 3614, 5892, 0, 4682, 880, 1708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27980 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[6627, 2, 5, 1, 4432, 3536, 352, 1429] payload_rows_per_dst=[13254, 4788, 10, 4082, 2912, 4160, 704, 2858] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13254, 4788, 10, 4082, 2912, 4160, 704, 2858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24172 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[7404, 0, 1, 0, 4297, 3417, 404, 861] payload_rows_per_dst=[0, 3058, 10822, 928, 8594, 5108, 808, 3448] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 3058, 10822, 928, 8594, 5108, 808, 3448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32078 payload_rows_remote=21552 dup_rows=690 dup_rows_remote=418 dedup_ratio=0.021057 router_tokens_per_expert=[6494, 1, 0, 1, 6365, 2282, 413, 828] payload_rows_per_dst=[0, 2, 0, 1272, 2954, 4186, 13138, 10526] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 418, 272] combine_rows_per_src=[0, 2, 0, 1272, 2954, 4186, 13138, 10526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=244039680 dispatch_meta_bytes=119600 combine_payload_bytes=244039680 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=244159280 ingress_bytes=3062611968 link_bytes=3062611968 token_bytes=488198960 weight_movement_bytes=2818572288 total_nvlink_bytes=3306771248 weight_to_token_ratio=5.773409 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.714934862e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=30210 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[6764, 1, 0, 0, 5718, 2828, 415, 658] payload_rows_per_dst=[0, 8064, 6348, 2556, 7996, 5656, 830, 1316] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8064, 6348, 2556, 7996, 5656, 830, 1316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=238829568 dispatch_meta_bytes=116616 combine_payload_bytes=238829568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=238946184 ingress_bytes=1648115712 link_bytes=1648115712 token_bytes=477775752 weight_movement_bytes=1409286144 total_nvlink_bytes=1887061896 weight_to_token_ratio=2.949681 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.228760030e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=229212160 dispatch_meta_bytes=111920 combine_payload_bytes=229212160 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=229324080 ingress_bytes=1638498304 link_bytes=1638498304 token_bytes=458536240 weight_movement_bytes=1409286144 total_nvlink_bytes=1867822384 weight_to_token_ratio=3.073446 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.174906560e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32078 payload_rows_remote=20594 dup_rows=690 dup_rows_remote=346 dedup_ratio=0.021057 router_tokens_per_expert=[7031, 0, 1, 0, 5502, 2703, 412, 735] payload_rows_per_dst=[0, 0, 4766, 6924, 0, 2512, 11484, 6392] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 344, 346] combine_rows_per_src=[0, 0, 4766, 6924, 0, 2512, 11484, 6392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=198017024 dispatch_meta_bytes=96696 combine_payload_bytes=198017024 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=198113720 ingress_bytes=4425875456 link_bytes=4425875456 token_bytes=396130744 weight_movement_bytes=4227858432 total_nvlink_bytes=4623989176 weight_to_token_ratio=10.672886 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6784 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.478305510e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=176553984 dispatch_meta_bytes=87880 combine_payload_bytes=176553984 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=176641864 ingress_bytes=1585840128 link_bytes=1585840128 token_bytes=353195848 weight_movement_bytes=1409286144 total_nvlink_bytes=1762481992 weight_to_token_ratio=3.990098 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3712 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.880042755e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31472 payload_rows_remote=16890 dup_rows=1296 dup_rows_remote=0 dedup_ratio=0.039551 router_tokens_per_expert=[7197, 0, 5, 2, 4171, 3631, 525, 853] payload_rows_per_dst=[14582, 4494, 10, 3852, 2182, 3596, 1050, 1706] dup_rows_per_dst=[1296, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[14582, 4494, 10, 3852, 2182, 3596, 1050, 1706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=247480320 dispatch_meta_bytes=120848 combine_payload_bytes=247480320 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=247601168 ingress_bytes=1656766464 link_bytes=1656766464 token_bytes=495081488 weight_movement_bytes=1409286144 total_nvlink_bytes=1904367632 weight_to_token_ratio=2.846574 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.277200630e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=168706048 dispatch_meta_bytes=83760 combine_payload_bytes=168706048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=168789808 ingress_bytes=1577992192 link_bytes=1577992192 token_bytes=337495856 weight_movement_bytes=1409286144 total_nvlink_bytes=1746782000 weight_to_token_ratio=4.175714 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1664 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.836097589e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=138362880 dispatch_meta_bytes=67560 combine_payload_bytes=138362880 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=138430440 ingress_bytes=5775507456 link_bytes=5775507456 token_bytes=276793320 weight_movement_bytes=5637144576 total_nvlink_bytes=5913937896 weight_to_token_ratio=20.365898 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.234043094e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 9, 7, 8172, 5, 0, 8191, 0] payload_rows_per_dst=[60, 8118, 8204, 8190, 10, 8186, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[60, 8118, 8204, 8190, 10, 8186, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=24544 dup_rows=34 dup_rows_remote=0 dedup_ratio=0.001038 router_tokens_per_expert=[0, 5, 2, 8166, 3, 20, 8188, 0] payload_rows_per_dst=[0, 10, 4, 8188, 8146, 8190, 8188, 8] dup_rows_per_dst=[0, 0, 0, 0, 0, 34, 0, 0] combine_rows_per_src=[0, 10, 4, 8188, 8146, 8190, 8188, 8] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32758 payload_rows_remote=24584 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000305 router_tokens_per_expert=[0, 12, 5, 8175, 0, 0, 8192, 0] payload_rows_per_dst=[16360, 8174, 8154, 70, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 10, 0, 0, 0, 0, 0] combine_rows_per_src=[16360, 8174, 8154, 70, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8, 0, 8184, 0, 0, 8192, 0] payload_rows_per_dst=[0, 50, 0, 8190, 8180, 8156, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 50, 0, 8190, 8180, 8156, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 4, 3, 8185, 0, 0, 8191, 0] payload_rows_per_dst=[2, 8, 6, 0, 8180, 0, 8192, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8, 6, 0, 8180, 0, 8192, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201228288 dispatch_meta_bytes=98256 combine_payload_bytes=201228288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201326544 ingress_bytes=1610514432 link_bytes=1610514432 token_bytes=402554832 weight_movement_bytes=1409286144 total_nvlink_bytes=1811840976 weight_to_token_ratio=3.500855 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018208557e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201064448 dispatch_meta_bytes=98176 combine_payload_bytes=201064448 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201162624 ingress_bytes=1610350592 link_bytes=1610350592 token_bytes=402227072 weight_movement_bytes=1409286144 total_nvlink_bytes=1811513216 weight_to_token_ratio=3.503708 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.017291122e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=24630 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000244 router_tokens_per_expert=[0, 1, 4, 8181, 0, 6, 8192, 0] payload_rows_per_dst=[0, 8178, 8196, 8130, 60, 8192, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 8, 0, 0] combine_rows_per_src=[0, 8178, 8196, 8130, 60, 8192, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98376 combine_payload_bytes=201392128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201490504 ingress_bytes=1610678272 link_bytes=1610678272 token_bytes=402882632 weight_movement_bytes=1409286144 total_nvlink_bytes=1812168776 weight_to_token_ratio=3.498007 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019125993e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201424896 dispatch_meta_bytes=98352 combine_payload_bytes=201424896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201523248 ingress_bytes=1610711040 link_bytes=1610711040 token_bytes=402948144 weight_movement_bytes=1409286144 total_nvlink_bytes=1812234288 weight_to_token_ratio=3.497438 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019309480e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 3, 1, 8186, 2, 1, 8191, 0] payload_rows_per_dst=[0, 6, 2, 0, 8190, 2, 8192, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 6, 2, 0, 8190, 2, 8192, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[1, 2, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16422 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 23, 0, 8167, 1, 0, 8190, 2] payload_rows_per_dst=[16346, 8224, 8192, 0, 2, 0, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16346, 8224, 8192, 0, 2, 0, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 2, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201768960 dispatch_meta_bytes=98552 combine_payload_bytes=201768960 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=201867512 ingress_bytes=5838913536 link_bytes=5838913536 token_bytes=403636472 weight_movement_bytes=5637144576 total_nvlink_bytes=6040781048 weight_to_token_ratio=13.965895 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269547852e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=201424896 ingress_bytes=5838471168 link_bytes=5838471168 token_bytes=402751488 weight_movement_bytes=5637144576 total_nvlink_bytes=6039896064 weight_to_token_ratio=13.996583 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269300144e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24596 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 0, 0, 1, 0, 3, 0, 8190] payload_rows_per_dst=[22, 8192, 8172, 16376, 0, 6, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[22, 8192, 8172, 16376, 0, 6, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24568 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8184, 2, 3, 8, 0, 4, 1, 8182] payload_rows_per_dst=[0, 4, 6, 16, 16366, 8200, 8174, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 6, 16, 16366, 8200, 8174, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134529024 dispatch_meta_bytes=65688 combine_payload_bytes=134529024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134594712 ingress_bytes=1543815168 link_bytes=1543815168 token_bytes=269123736 weight_movement_bytes=1409286144 total_nvlink_bytes=1678409880 weight_to_token_ratio=5.236573 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644720521e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 20, 0, 16384, 8172, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 20, 0, 16384, 8172, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8181, 0, 0, 0, 0, 0, 11, 8192] payload_rows_per_dst=[16362, 8192, 8188, 4, 0, 0, 22, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16362, 8192, 8188, 4, 0, 0, 22, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 1, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 2, 0, 8192, 8190, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 0, 8192, 8190, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201490432 dispatch_meta_bytes=98384 combine_payload_bytes=201490432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201588816 ingress_bytes=1610776576 link_bytes=1610776576 token_bytes=403079248 weight_movement_bytes=1409286144 total_nvlink_bytes=1812365392 weight_to_token_ratio=3.496300 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019676454e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201261056 dispatch_meta_bytes=98272 combine_payload_bytes=201261056 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201359328 ingress_bytes=1610547200 link_bytes=1610547200 token_bytes=402620384 weight_movement_bytes=1409286144 total_nvlink_bytes=1811906528 weight_to_token_ratio=3.500285 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018392045e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24596 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 1, 1, 0, 0, 8191] payload_rows_per_dst=[0, 0, 0, 2, 14, 8198, 8172, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 14, 8198, 8172, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 2, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16402 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 1, 8191] payload_rows_per_dst=[0, 8188, 8192, 16366, 4, 0, 18, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8188, 8192, 16366, 4, 0, 18, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=16384 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000183 router_tokens_per_expert=[8189, 3, 2, 1, 0, 0, 0, 8189] payload_rows_per_dst=[16378, 8192, 8190, 2, 0, 0, 0, 0] dup_rows_per_dst=[6, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 8192, 8190, 2, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134283264 ingress_bytes=5066719232 link_bytes=5066719232 token_bytes=268500992 weight_movement_bytes=4932501504 total_nvlink_bytes=5201002496 weight_to_token_ratio=18.370515 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837151275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201490432 dispatch_meta_bytes=98384 combine_payload_bytes=201490432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201588816 ingress_bytes=1610776576 link_bytes=1610776576 token_bytes=403079248 weight_movement_bytes=1409286144 total_nvlink_bytes=1812365392 weight_to_token_ratio=3.496300 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019676454e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 1, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[12, 8196, 8180, 0, 8188, 8192, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12, 8196, 8180, 0, 8188, 8192, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134365184 dispatch_meta_bytes=65608 combine_payload_bytes=134365184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134430792 ingress_bytes=1543651328 link_bytes=1543651328 token_bytes=268795976 weight_movement_bytes=1409286144 total_nvlink_bytes=1678082120 weight_to_token_ratio=5.242958 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643803085e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=32760 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[8185, 0, 2, 6, 1, 2, 3, 8185] payload_rows_per_dst=[0, 0, 4, 16378, 2, 4, 16374, 2] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 16378, 2, 4, 16374, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=5066719232 link_bytes=5066719232 token_bytes=268500992 weight_movement_bytes=4932501504 total_nvlink_bytes=5201002496 weight_to_token_ratio=18.370515 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837151275e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 0, 1, 1, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 12, 16372, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 12, 16372, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 1, 0, 0, 0, 0, 0, 8191] payload_rows_per_dst=[16384, 8194, 8190, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8194, 8190, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201424896 dispatch_meta_bytes=98352 combine_payload_bytes=201424896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201523248 ingress_bytes=1610711040 link_bytes=1610711040 token_bytes=402948144 weight_movement_bytes=1409286144 total_nvlink_bytes=1812234288 weight_to_token_ratio=3.497438 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019309480e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131056 combine_payload_bytes=268369920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268500976 ingress_bytes=1677656064 link_bytes=1677656064 token_bytes=536870896 weight_movement_bytes=1409286144 total_nvlink_bytes=1946157040 weight_to_token_ratio=2.625000 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394173670e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16380 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 8192, 8188, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[0, 0, 0, 0, 8192, 8188, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 1, 0, 0, 1, 8191] payload_rows_per_dst=[0, 0, 0, 2, 8192, 8190, 6, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 8192, 8190, 6, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 0, 0, 1, 0, 8192] payload_rows_per_dst=[0, 8190, 8190, 2, 8190, 8194, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8190, 8190, 2, 8190, 8194, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134184960 dispatch_meta_bytes=65520 combine_payload_bytes=134184960 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134250480 ingress_bytes=5066686464 link_bytes=5066686464 token_bytes=268435440 weight_movement_bytes=4932501504 total_nvlink_bytes=5200936944 weight_to_token_ratio=18.375001 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837132926e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8186, 0, 0, 5, 2, 0, 1, 8190] payload_rows_per_dst=[16372, 8188, 8192, 10, 4, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16372, 8188, 8192, 10, 4, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268517352 ingress_bytes=1677672448 link_bytes=1677672448 token_bytes=536903656 weight_movement_bytes=1409286144 total_nvlink_bytes=1946189800 weight_to_token_ratio=2.624840 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394265414e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16420 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 10, 3, 0, 8181, 8189, 0] payload_rows_per_dst=[34, 2, 16348, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[34, 2, 16348, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24584 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 5, 1, 0, 0, 8187, 8187, 4] payload_rows_per_dst=[0, 10, 2, 0, 16372, 8182, 8190, 10] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[0, 10, 2, 0, 16372, 8182, 8190, 10] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134381616 ingress_bytes=5066817536 link_bytes=5066817536 token_bytes=268697648 weight_movement_bytes=4932501504 total_nvlink_bytes=5201199152 weight_to_token_ratio=18.357070 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837206321e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16398 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 12, 3, 0, 8177, 8191, 0] payload_rows_per_dst=[16338, 16370, 24, 36, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16338, 16370, 24, 36, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 4, 8, 16376, 8188, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 8, 16376, 8188, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134512640 dispatch_meta_bytes=65680 combine_payload_bytes=134512640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134578320 ingress_bytes=1543798784 link_bytes=1543798784 token_bytes=269090960 weight_movement_bytes=1409286144 total_nvlink_bytes=1678377104 weight_to_token_ratio=5.237211 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644628777e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 0, 8191, 8192, 0] payload_rows_per_dst=[0, 0, 2, 0, 0, 8190, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 0, 8190, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98336 combine_payload_bytes=201392128 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=201490464 ingress_bytes=5838536704 link_bytes=5838536704 token_bytes=402882592 weight_movement_bytes=5637144576 total_nvlink_bytes=6040027168 weight_to_token_ratio=13.992028 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269336842e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 0, 0, 8191, 8191, 0] payload_rows_per_dst=[0, 0, 4, 0, 8, 8194, 8192, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 0, 8, 8194, 8192, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134332416 dispatch_meta_bytes=65592 combine_payload_bytes=134332416 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134398008 ingress_bytes=1543618560 link_bytes=1543618560 token_bytes=268730424 weight_movement_bytes=1409286144 total_nvlink_bytes=1678016568 weight_to_token_ratio=5.244237 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643619598e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=16416 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000366 router_tokens_per_expert=[11, 0, 7, 0, 0, 8174, 8192, 0] payload_rows_per_dst=[22, 0, 16368, 16340, 12, 12, 2, 0] dup_rows_per_dst=[0, 0, 12, 0, 0, 0, 0, 0] combine_rows_per_src=[22, 0, 16368, 16340, 12, 12, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16392 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 3, 2, 0, 0, 8187, 8190, 2] payload_rows_per_dst=[16374, 16384, 4, 0, 0, 0, 0, 4] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16374, 16384, 4, 0, 0, 0, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=201424896 ingress_bytes=5838471168 link_bytes=5838471168 token_bytes=402751488 weight_movement_bytes=5637144576 total_nvlink_bytes=6039896064 weight_to_token_ratio=13.996583 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269300144e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[27, 0, 0, 0, 0, 8165, 8192, 0] payload_rows_per_dst=[54, 6, 16378, 8176, 8154, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[54, 6, 16378, 8176, 8154, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134479872 dispatch_meta_bytes=65712 combine_payload_bytes=134479872 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134545584 ingress_bytes=1543766016 link_bytes=1543766016 token_bytes=269025456 weight_movement_bytes=1409286144 total_nvlink_bytes=1678311600 weight_to_token_ratio=5.238486 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644445290e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65576 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348840 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632104 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918248 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17268 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[91, 0, 5, 2, 0, 8099, 8185, 2] payload_rows_per_dst=[182, 0, 10, 8180, 8176, 15500, 4, 716] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[182, 0, 10, 8180, 8176, 15500, 4, 716] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=16388 dup_rows=12 dup_rows_remote=0 dedup_ratio=0.000366 router_tokens_per_expert=[123, 6, 0, 0, 0, 8063, 8192, 0] payload_rows_per_dst=[15346, 16368, 16, 280, 746, 0, 0, 0] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15346, 16368, 16, 280, 746, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[41, 0, 0, 0, 0, 8151, 8192, 0] payload_rows_per_dst=[82, 0, 0, 8192, 8192, 16302, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[82, 0, 0, 8192, 8192, 16302, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17060 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[338, 2, 0, 0, 0, 7854, 8190, 0] payload_rows_per_dst=[676, 4, 0, 0, 0, 0, 16380, 15708] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[676, 4, 0, 0, 0, 0, 16380, 15708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=141459456 dispatch_meta_bytes=69072 combine_payload_bytes=141459456 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=141528528 ingress_bytes=5778604032 link_bytes=5778604032 token_bytes=282987984 weight_movement_bytes=5637144576 total_nvlink_bytes=5920132560 weight_to_token_ratio=19.920085 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.235777047e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[23, 0, 0, 0, 0, 8169, 8192, 0] payload_rows_per_dst=[46, 0, 0, 0, 0, 0, 16384, 16338] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[46, 0, 0, 0, 0, 0, 16384, 16338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24828 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 0, 0, 0, 0, 8189, 8192, 0] payload_rows_per_dst=[6, 0, 16356, 7940, 7500, 966, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6, 0, 16356, 7940, 7500, 966, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 1, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[91, 0, 4, 0, 0, 8097, 8189, 3] payload_rows_per_dst=[16376, 16378, 8, 0, 0, 0, 0, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 16378, 8, 0, 0, 0, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=139755520 dispatch_meta_bytes=68240 combine_payload_bytes=139755520 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=139823760 ingress_bytes=1549041664 link_bytes=1549041664 token_bytes=279579280 weight_movement_bytes=1409286144 total_nvlink_bytes=1688865424 weight_to_token_ratio=5.040739 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.673986717e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134283264 ingress_bytes=5771362304 link_bytes=5771362304 token_bytes=268500992 weight_movement_bytes=5637144576 total_nvlink_bytes=5905645568 weight_to_token_ratio=20.994874 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231721982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=203390976 dispatch_meta_bytes=99312 combine_payload_bytes=203390976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=203490288 ingress_bytes=1612677120 link_bytes=1612677120 token_bytes=406881264 weight_movement_bytes=1409286144 total_nvlink_bytes=1816167408 weight_to_token_ratio=3.463630 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.030318707e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=16386 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000366 router_tokens_per_expert=[1, 8191, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[4, 20, 16370, 16362, 0, 0, 0, 0] dup_rows_per_dst=[2, 10, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 20, 16370, 16362, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16516 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8182, 0, 8, 4, 4, 2, 8184] payload_rows_per_dst=[0, 0, 0, 16, 16376, 16252, 124, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16, 16376, 16252, 124, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8187, 0, 3, 0, 2, 0, 8192] payload_rows_per_dst=[16384, 16374, 0, 6, 0, 4, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16374, 0, 6, 0, 4, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65592 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299704 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533816 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819960 weight_to_token_ratio=5.248077 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8188, 0, 0, 0, 4, 0, 8192] payload_rows_per_dst=[0, 0, 18, 0, 16384, 16366, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 18, 0, 16384, 16366, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=135299072 dispatch_meta_bytes=66064 combine_payload_bytes=135299072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135365136 ingress_bytes=1544585216 link_bytes=1544585216 token_bytes=270664208 weight_movement_bytes=1409286144 total_nvlink_bytes=1679950352 weight_to_token_ratio=5.206770 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.649032468e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8131, 0, 0, 0, 61, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 122, 16262, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 122, 16262, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8186, 0, 0, 0, 6, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 12, 16372, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 12, 16372, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134365224 ingress_bytes=5771444224 link_bytes=5771444224 token_bytes=268664872 weight_movement_bytes=5637144576 total_nvlink_bytes=5905809448 weight_to_token_ratio=20.982068 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231767854e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16384 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8188, 0, 1, 0, 3, 0, 8192] payload_rows_per_dst=[0, 0, 16376, 16382, 2, 6, 0, 0] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16376, 16382, 2, 6, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8182, 2, 0, 3, 3, 5, 8189] payload_rows_per_dst=[16378, 16364, 4, 0, 6, 6, 10, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 16364, 4, 0, 6, 6, 10, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134283264 ingress_bytes=5771362304 link_bytes=5771362304 token_bytes=268500992 weight_movement_bytes=5637144576 total_nvlink_bytes=5905645568 weight_to_token_ratio=20.994874 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231721982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[8, 16376, 0, 8, 16376, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 16376, 0, 8, 16376, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16398 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8186, 0, 3, 0, 3, 8185, 3, 4] payload_rows_per_dst=[0, 0, 8186, 8190, 6, 16370, 8, 8] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8186, 8190, 6, 16370, 8, 8] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 0, 8192, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 8192, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 8, 8184, 8192, 0, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8, 8184, 8192, 0, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134332416 dispatch_meta_bytes=65592 combine_payload_bytes=134332416 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134398008 ingress_bytes=4362190848 link_bytes=4362190848 token_bytes=268730424 weight_movement_bytes=4227858432 total_nvlink_bytes=4496588856 weight_to_token_ratio=15.732712 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442644788e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 1, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 2, 8, 0, 0, 12, 16374, 16372] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 8, 0, 0, 12, 16374, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 16382, 0, 2, 16382, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16382, 0, 2, 16382, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8188, 0, 3, 1, 2, 8187, 1, 2] payload_rows_per_dst=[16376, 0, 8198, 8184, 4, 0, 2, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 0, 8198, 8184, 4, 0, 2, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134365224 ingress_bytes=1543585792 link_bytes=1543585792 token_bytes=268664872 weight_movement_bytes=1409286144 total_nvlink_bytes=1677951016 weight_to_token_ratio=5.245517 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643436111e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 8192, 0, 0, 0, 8191, 0] payload_rows_per_dst=[8, 2, 16376, 0, 16382, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 2, 16376, 0, 16382, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16424 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 17, 8185, 5, 4, 0, 8173, 0] payload_rows_per_dst=[0, 34, 0, 16380, 8, 16344, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 34, 0, 16380, 8, 16344, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134348832 ingress_bytes=4362141696 link_bytes=4362141696 token_bytes=268632096 weight_movement_bytes=4227858432 total_nvlink_bytes=4496490528 weight_to_token_ratio=15.738471 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442617265e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16416 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16352, 4, 0, 28, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16352, 4, 0, 28, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134348832 ingress_bytes=5066784768 link_bytes=5066784768 token_bytes=268632096 weight_movement_bytes=4932501504 total_nvlink_bytes=5201133600 weight_to_token_ratio=18.361549 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837187972e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 4, 16380, 0, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 16380, 0, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 1, 8191, 0] payload_rows_per_dst=[0, 0, 0, 2, 0, 2, 16382, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 0, 2, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134545408 dispatch_meta_bytes=65696 combine_payload_bytes=134545408 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134611104 ingress_bytes=1543831552 link_bytes=1543831552 token_bytes=269156512 weight_movement_bytes=1409286144 total_nvlink_bytes=1678442656 weight_to_token_ratio=5.235936 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644812264e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134479872 dispatch_meta_bytes=65664 combine_payload_bytes=134479872 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134545536 ingress_bytes=1543766016 link_bytes=1543766016 token_bytes=269025408 weight_movement_bytes=1409286144 total_nvlink_bytes=1678311552 weight_to_token_ratio=5.238487 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644445290e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 16384, 0, 16346, 38, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 0, 16346, 38, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 14, 8188, 3, 2, 0, 8176, 1] payload_rows_per_dst=[16376, 16380, 0, 6, 4, 0, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 16380, 0, 6, 4, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134299656 ingress_bytes=5066735616 link_bytes=5066735616 token_bytes=268533768 weight_movement_bytes=4932501504 total_nvlink_bytes=5201035272 weight_to_token_ratio=18.368273 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837160449e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 10, 16384, 2, 8188, 8184, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 10, 16384, 2, 8188, 8184, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5, 0, 0, 8186, 1, 6, 1, 8185] payload_rows_per_dst=[10, 0, 0, 16372, 2, 12, 16372, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10, 0, 0, 16372, 2, 12, 16372, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 2, 8190] payload_rows_per_dst=[16376, 16380, 8, 0, 0, 0, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 16380, 8, 0, 0, 0, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 16384, 4, 0, 16380, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 4, 0, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268337152 dispatch_meta_bytes=131024 combine_payload_bytes=268337152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268468176 ingress_bytes=1677623296 link_bytes=1677623296 token_bytes=536805328 weight_movement_bytes=1409286144 total_nvlink_bytes=1946091472 weight_to_token_ratio=2.625321 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393990183e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 2, 8190, 8192, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 8190, 8192, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 8192, 8192, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 8192, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 16376, 8, 8192, 8180, 12, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16376, 8, 8192, 8180, 12, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 0, 0, 8188, 0, 4, 0, 8189] payload_rows_per_dst=[16382, 16378, 0, 0, 0, 8, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16378, 0, 0, 0, 8, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134283264 ingress_bytes=5066719232 link_bytes=5066719232 token_bytes=268500992 weight_movement_bytes=4932501504 total_nvlink_bytes=5201002496 weight_to_token_ratio=18.370515 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837151275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 2, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[4, 8188, 8192, 8192, 12, 8180, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 8188, 8192, 8192, 12, 8180, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131040 combine_payload_bytes=268369920 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=268500960 ingress_bytes=5200871424 link_bytes=5200871424 token_bytes=536870880 weight_movement_bytes=4932501504 total_nvlink_bytes=5469372384 weight_to_token_ratio=9.187501 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912270902e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8185, 7, 0, 8182, 6, 3, 0] payload_rows_per_dst=[2, 0, 8188, 4, 8190, 8204, 8180, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 8188, 4, 8190, 8204, 8180, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24576 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16378, 8190, 2, 4, 8192, 0, 0, 0] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 8190, 2, 4, 8192, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 8191, 0, 0, 8191, 0, 0, 0] payload_rows_per_dst=[4, 4, 8188, 0, 8190, 8190, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 4, 8188, 0, 8190, 8190, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 8191, 0, 0, 8191, 0, 0, 0] payload_rows_per_dst=[4, 0, 0, 8190, 0, 0, 8192, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 0, 8190, 0, 0, 8192, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 8190, 2, 0, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8190, 2, 0, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201228288 dispatch_meta_bytes=98256 combine_payload_bytes=201228288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201326544 ingress_bytes=1610514432 link_bytes=1610514432 token_bytes=402554832 weight_movement_bytes=1409286144 total_nvlink_bytes=1811840976 weight_to_token_ratio=3.500855 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018208557e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=201424896 ingress_bytes=5838471168 link_bytes=5838471168 token_bytes=402751488 weight_movement_bytes=5637144576 total_nvlink_bytes=6039896064 weight_to_token_ratio=13.996583 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269300144e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=201441288 ingress_bytes=5838487552 link_bytes=5838487552 token_bytes=402784264 weight_movement_bytes=5637144576 total_nvlink_bytes=6039928840 weight_to_token_ratio=13.995444 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269309319e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 8192, 8190, 8188, 4, 8190, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8190, 8188, 4, 8190, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16392 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[1, 8187, 4, 0, 8185, 2, 4, 1] payload_rows_per_dst=[16374, 8192, 8, 0, 8178, 4, 8, 2] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16374, 8192, 8, 0, 8178, 4, 8, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98320 combine_payload_bytes=201359360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201457680 ingress_bytes=1610645504 link_bytes=1610645504 token_bytes=402817040 weight_movement_bytes=1409286144 total_nvlink_bytes=1812103184 weight_to_token_ratio=3.498576 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018942506e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[8, 10, 16374, 8184, 8192, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 10, 16374, 8184, 8192, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7, 8169, 6, 17, 1, 0, 8184] payload_rows_per_dst=[0, 14, 0, 8204, 8180, 2, 16368, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 14, 0, 8204, 8180, 2, 16368, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24586 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 10, 8192, 8182, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 10, 8192, 8182, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 2, 16382, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 2, 16382, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134365224 ingress_bytes=4362158080 link_bytes=4362158080 token_bytes=268664872 weight_movement_bytes=4227858432 total_nvlink_bytes=4496523304 weight_to_token_ratio=15.736551 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442626439e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 8178, 3, 10, 0, 3, 8188] payload_rows_per_dst=[16376, 16360, 0, 6, 20, 0, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 16360, 0, 6, 20, 0, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24586 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 16384, 8182, 8192, 0, 10, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 8182, 8192, 0, 10, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201408512 dispatch_meta_bytes=98344 combine_payload_bytes=201408512 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201506856 ingress_bytes=1610694656 link_bytes=1610694656 token_bytes=402915368 weight_movement_bytes=1409286144 total_nvlink_bytes=1812201512 weight_to_token_ratio=3.497722 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019217737e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[4, 8, 16380, 16376, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 8, 16380, 16376, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201408512 dispatch_meta_bytes=98344 combine_payload_bytes=201408512 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201506856 ingress_bytes=1610694656 link_bytes=1610694656 token_bytes=402915368 weight_movement_bytes=1409286144 total_nvlink_bytes=1812201512 weight_to_token_ratio=3.497722 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019217737e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 0, 0, 8, 8185, 0, 5, 8184] payload_rows_per_dst=[4, 0, 0, 16, 16364, 6, 16378, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 0, 16, 16364, 6, 16378, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[0, 0, 4, 0, 16384, 0, 16380, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 0, 16384, 0, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 8191, 0, 0, 8192] payload_rows_per_dst=[0, 0, 2, 0, 0, 16382, 4, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 0, 16382, 4, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268517352 ingress_bytes=1677672448 link_bytes=1677672448 token_bytes=536903656 weight_movement_bytes=1409286144 total_nvlink_bytes=1946189800 weight_to_token_ratio=2.624840 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394265414e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16380 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 16380, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[0, 0, 0, 0, 0, 16380, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 5, 8188, 0, 3, 8187] payload_rows_per_dst=[16376, 16376, 0, 10, 0, 0, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 16376, 0, 10, 0, 0, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16402 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[0, 0, 16382, 16366, 20, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16382, 16366, 20, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134283264 ingress_bytes=5066719232 link_bytes=5066719232 token_bytes=268500992 weight_movement_bytes=4932501504 total_nvlink_bytes=5201002496 weight_to_token_ratio=18.370515 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837151275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134184960 dispatch_meta_bytes=65520 combine_payload_bytes=134184960 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134250480 ingress_bytes=5066686464 link_bytes=5066686464 token_bytes=268435440 weight_movement_bytes=4932501504 total_nvlink_bytes=5200936944 weight_to_token_ratio=18.375001 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837132926e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 8202, 8192, 0, 8192, 8182, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8202, 8192, 0, 8192, 8182, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134365184 dispatch_meta_bytes=65608 combine_payload_bytes=134365184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134430792 ingress_bytes=1543651328 link_bytes=1543651328 token_bytes=268795976 weight_movement_bytes=1409286144 total_nvlink_bytes=1678082120 weight_to_token_ratio=5.242958 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643803085e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24562 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[6, 8183, 0, 0, 8186, 7, 0, 2] payload_rows_per_dst=[12, 0, 0, 16372, 0, 8206, 8174, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12, 0, 0, 16372, 0, 8206, 8174, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24574 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16374, 8192, 8200, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16374, 8192, 8200, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 16384, 0, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 8192, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201211904 dispatch_meta_bytes=98248 combine_payload_bytes=201211904 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201310152 ingress_bytes=1610498048 link_bytes=1610498048 token_bytes=402522056 weight_movement_bytes=1409286144 total_nvlink_bytes=1811808200 weight_to_token_ratio=3.501140 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018116814e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24570 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8192, 0, 8198, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 0, 8198, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=201408504 ingress_bytes=5133811712 link_bytes=5133811712 token_bytes=402718712 weight_movement_bytes=4932501504 total_nvlink_bytes=5335220216 weight_to_token_ratio=12.248007 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874720263e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8192, 0, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 0, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 8186, 0, 2, 8188, 4, 0, 1] payload_rows_per_dst=[16382, 8180, 8192, 4, 0, 8, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8180, 8192, 4, 0, 8, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 8192, 8184, 8, 8192, 8180, 12, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8184, 8, 8192, 8180, 12, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=268566528 ingress_bytes=5200936960 link_bytes=5200936960 token_bytes=537001984 weight_movement_bytes=4932501504 total_nvlink_bytes=5469503488 weight_to_token_ratio=9.185258 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912307600e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201277440 dispatch_meta_bytes=98280 combine_payload_bytes=201277440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201375720 ingress_bytes=1610563584 link_bytes=1610563584 token_bytes=402653160 weight_movement_bytes=1409286144 total_nvlink_bytes=1811939304 weight_to_token_ratio=3.500000 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018483788e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[4, 8188, 8208, 0, 16368, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 8188, 8208, 0, 16368, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131040 combine_payload_bytes=268369920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268500960 ingress_bytes=1677656064 link_bytes=1677656064 token_bytes=536870880 weight_movement_bytes=1409286144 total_nvlink_bytes=1946157024 weight_to_token_ratio=2.625000 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394173670e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24574 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[8184, 0, 0, 3, 5, 8, 8183, 1] payload_rows_per_dst=[0, 0, 0, 16370, 10, 8192, 8192, 2] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16370, 10, 8192, 8192, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 8192, 8192, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201195520 dispatch_meta_bytes=98240 combine_payload_bytes=201195520 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201293760 ingress_bytes=1610481664 link_bytes=1610481664 token_bytes=402489280 weight_movement_bytes=1409286144 total_nvlink_bytes=1811775424 weight_to_token_ratio=3.501425 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018025070e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98304 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408512 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718720 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004864 weight_to_token_ratio=3.499430 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 4, 0, 16380, 6, 8186, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 0, 16380, 6, 8186, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 14, 0, 8192, 8192, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 14, 0, 8192, 8192, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=5702925768 prefetch_bytes=20434649088 reduce_bytes=8455716864 weight_movement_bytes=28890365952 egress_bytes=2852158920 ingress_bytes=31741132800 link_bytes=31741132800 total_nvlink_bytes=34593291720 weight_to_token_ratio=5.065885 prefetch_share_of_ingress=0.643791 dedup_ratio=0.000137 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=12 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.021780 gemm_inner_pad_upper_bound=6731 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.110856839e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 1, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 2, 0, 0, 8190, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 0, 8190, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16388 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[8188, 0, 0, 0, 0, 4, 8186, 6] payload_rows_per_dst=[16376, 8192, 8176, 0, 0, 8, 0, 12] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 8192, 8176, 0, 0, 8, 0, 12] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 2, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6533209392 prefetch_bytes=19730006016 reduce_bytes=16911433728 weight_movement_bytes=36641439744 egress_bytes=3267403056 ingress_bytes=39907246080 link_bytes=39907246080 total_nvlink_bytes=43174649136 weight_to_token_ratio=5.608490 prefetch_share_of_ingress=0.494397 dedup_ratio=0.001064 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=24 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.021307 gemm_inner_pad_upper_bound=6985 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.396649500e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 8192, 8190, 2, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8190, 2, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268517352 ingress_bytes=1677672448 link_bytes=1677672448 token_bytes=536903656 weight_movement_bytes=1409286144 total_nvlink_bytes=1946189800 weight_to_token_ratio=2.624840 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394265414e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 2, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201424896 ingress_bytes=5133828096 link_bytes=5133828096 token_bytes=402751488 weight_movement_bytes=4932501504 total_nvlink_bytes=5335252992 weight_to_token_ratio=12.247010 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874729437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=5716822888 prefetch_bytes=21139292160 reduce_bytes=13388218368 weight_movement_bytes=34527510528 egress_bytes=2859109224 ingress_bytes=37385224192 link_bytes=37385224192 total_nvlink_bytes=40244333416 weight_to_token_ratio=6.039633 prefetch_share_of_ingress=0.565445 dedup_ratio=0.000122 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=19 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.022254 gemm_inner_pad_upper_bound=6477 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.308385313e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134316048 ingress_bytes=5066752000 link_bytes=5066752000 token_bytes=268566544 weight_movement_bytes=4932501504 total_nvlink_bytes=5201068048 weight_to_token_ratio=18.366031 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837169623e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6740876528 prefetch_bytes=19025362944 reduce_bytes=18320719872 weight_movement_bytes=37346082816 egress_bytes=3371261168 ingress_bytes=40715698176 link_bytes=40715698176 total_nvlink_bytes=44086959344 weight_to_token_ratio=5.540241 prefetch_share_of_ingress=0.467273 dedup_ratio=0.000233 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=26 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023201 gemm_inner_pad_upper_bound=6604 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.424943214e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6310993720 prefetch_bytes=14797504512 reduce_bytes=28890365952 weight_movement_bytes=43687870464 egress_bytes=3156270904 ingress_bytes=46842593280 link_bytes=46842593280 total_nvlink_bytes=49998864184 weight_to_token_ratio=6.922503 prefetch_share_of_ingress=0.315898 dedup_ratio=0.004917 prefetch_slots=42 unused_prefetch_slots=22 slot_utilization=0.656250 reduce_slots=41 remote_demand=21 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.024148 gemm_inner_pad_upper_bound=7112 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.639368558e-04 prefetch_bytes_per_gemm_flop=5.178740530e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4442955672 prefetch_bytes=15502147584 reduce_bytes=29595009024 weight_movement_bytes=45097156608 egress_bytes=2222022552 ingress_bytes=47318089728 link_bytes=47318089728 total_nvlink_bytes=49540112280 weight_to_token_ratio=10.150260 prefetch_share_of_ingress=0.327616 dedup_ratio=0.004795 prefetch_slots=44 unused_prefetch_slots=20 slot_utilization=0.687500 reduce_slots=42 remote_demand=22 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023674 gemm_inner_pad_upper_bound=7112 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.656009693e-04 prefetch_bytes_per_gemm_flop=5.425347222e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6846644464 prefetch_bytes=21843935232 reduce_bytes=11978932224 weight_movement_bytes=33822867456 egress_bytes=3424157936 ingress_bytes=37245353984 link_bytes=37245353984 total_nvlink_bytes=40669511920 weight_to_token_ratio=4.940065 prefetch_share_of_ingress=0.586488 dedup_ratio=0.000134 prefetch_slots=62 unused_prefetch_slots=2 slot_utilization=0.968750 reduce_slots=17 remote_demand=31 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019886 gemm_inner_pad_upper_bound=6985 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.303490221e-04 prefetch_bytes_per_gemm_flop=7.644807449e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4445310712 prefetch_bytes=17616076800 reduce_bytes=22548578304 weight_movement_bytes=40164655104 egress_bytes=2223197944 ingress_bytes=42386767872 link_bytes=42386767872 total_nvlink_bytes=44609965816 weight_to_token_ratio=9.035286 prefetch_share_of_ingress=0.415603 dedup_ratio=0.002590 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=32 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.022254 gemm_inner_pad_upper_bound=7112 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.483426294e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none - [2026-09-16 21:34:41] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 604.0 | throughput per GPU (TFLOP/s/GPU): 551.9 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.014641E+01 | loss scale: 1.0 | grad norm: 15.677 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32334 payload_rows_remote=15972 dup_rows=434 dup_rows_remote=0 dedup_ratio=0.013245 router_tokens_per_expert=[218, 6, 867, 3449, 1945, 1347, 8180, 372] payload_rows_per_dst=[16362, 12, 8632, 0, 3890, 2694, 0, 744] dup_rows_per_dst=[434, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16362, 12, 8632, 0, 3890, 2694, 0, 744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31458 payload_rows_remote=28912 dup_rows=1310 dup_rows_remote=1310 dedup_ratio=0.039978 router_tokens_per_expert=[293, 1, 733, 4275, 1273, 1245, 8189, 375] payload_rows_per_dst=[586, 2, 1466, 8550, 2546, 8948, 8610, 750] dup_rows_per_dst=[0, 0, 0, 0, 0, 1310, 0, 0] combine_rows_per_src=[586, 2, 1466, 8550, 2546, 8948, 8610, 750] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31912 payload_rows_remote=26470 dup_rows=856 dup_rows_remote=856 dedup_ratio=0.026123 router_tokens_per_expert=[501, 1, 878, 3945, 1343, 1181, 8180, 355] payload_rows_per_dst=[11068, 5442, 9504, 140, 2686, 2362, 0, 710] dup_rows_per_dst=[854, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[11068, 5442, 9504, 140, 2686, 2362, 0, 710] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32020 payload_rows_remote=15654 dup_rows=748 dup_rows_remote=0 dedup_ratio=0.022827 router_tokens_per_expert=[246, 6, 669, 3888, 1657, 1361, 8175, 382] payload_rows_per_dst=[492, 12, 1338, 900, 7758, 2722, 2432, 16366] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 748] combine_rows_per_src=[492, 12, 1338, 900, 7758, 2722, 2432, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26410 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[425, 0, 1619, 3117, 1212, 1457, 8179, 375] payload_rows_per_dst=[850, 16358, 6358, 3114, 2424, 2914, 0, 750] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[850, 16358, 6358, 3114, 2424, 2914, 0, 750] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32114 payload_rows_remote=24822 dup_rows=654 dup_rows_remote=654 dedup_ratio=0.019958 router_tokens_per_expert=[360, 1, 1076, 3646, 1762, 1013, 8183, 343] payload_rows_per_dst=[720, 10942, 2152, 7292, 3524, 6798, 0, 686] dup_rows_per_dst=[0, 0, 0, 0, 0, 654, 0, 0] combine_rows_per_src=[720, 10942, 2152, 7292, 3524, 6798, 0, 686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32368 payload_rows_remote=27022 dup_rows=400 dup_rows_remote=400 dedup_ratio=0.012207 router_tokens_per_expert=[494, 0, 979, 3633, 1836, 979, 8191, 272] payload_rows_per_dst=[988, 0, 1958, 5878, 5060, 1958, 5346, 11180] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 400] combine_rows_per_src=[988, 0, 1958, 5878, 5060, 1958, 5346, 11180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30360 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[207, 0, 679, 3447, 2440, 1204, 8190, 217] payload_rows_per_dst=[414, 0, 1358, 6894, 4880, 2408, 16380, 434] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[414, 0, 1358, 6894, 4880, 2408, 16380, 434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=130842624 dispatch_meta_bytes=63888 combine_payload_bytes=130842624 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=130906512 ingress_bytes=1540128768 link_bytes=1540128768 token_bytes=261749136 weight_movement_bytes=1409286144 total_nvlink_bytes=1671035280 weight_to_token_ratio=5.384110 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1664 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.624078219e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=236847104 dispatch_meta_bytes=120888 combine_payload_bytes=236847104 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236967992 ingress_bytes=1646133248 link_bytes=1646133248 token_bytes=473815096 weight_movement_bytes=1409286144 total_nvlink_bytes=1883101240 weight_to_token_ratio=2.974338 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7168 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.217659059e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=216842240 dispatch_meta_bytes=109304 combine_payload_bytes=216842240 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=216951544 ingress_bytes=1626128384 link_bytes=1626128384 token_bytes=433793784 weight_movement_bytes=1409286144 total_nvlink_bytes=1843079928 weight_to_token_ratio=3.248747 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.105640171e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=128237568 dispatch_meta_bytes=62616 combine_payload_bytes=128237568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=128300184 ingress_bytes=1537523712 link_bytes=1537523712 token_bytes=256537752 weight_movement_bytes=1409286144 total_nvlink_bytes=1665823896 weight_to_token_ratio=5.493484 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.609490993e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=216350720 dispatch_meta_bytes=105640 combine_payload_bytes=216350720 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=216456360 ingress_bytes=1625636864 link_bytes=1625636864 token_bytes=432807080 weight_movement_bytes=1409286144 total_nvlink_bytes=1842093224 weight_to_token_ratio=3.256153 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4480 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.102887864e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=203341824 dispatch_meta_bytes=101904 combine_payload_bytes=203341824 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=203443728 ingress_bytes=4431200256 link_bytes=4431200256 token_bytes=406785552 weight_movement_bytes=4227858432 total_nvlink_bytes=4634643984 weight_to_token_ratio=10.393335 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.481287176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=221364224 dispatch_meta_bytes=109688 combine_payload_bytes=221364224 prefetch_bytes=1409286144 reduce_bytes=5637144576 egress_bytes=221473912 ingress_bytes=7267794944 link_bytes=7267794944 token_bytes=442838136 weight_movement_bytes=7046430720 total_nvlink_bytes=7489268856 weight_to_token_ratio=15.911978 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7680 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=4.069661796e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=248709120 dispatch_meta_bytes=121440 combine_payload_bytes=248709120 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=248830560 ingress_bytes=1657995264 link_bytes=1657995264 token_bytes=497539680 weight_movement_bytes=1409286144 total_nvlink_bytes=1906825824 weight_to_token_ratio=2.832510 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4992 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.284081397e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 3, 0, 0, 220, 8189, 7970, 2] payload_rows_per_dst=[16162, 8198, 7964, 0, 440, 0, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16162, 8198, 7964, 0, 440, 0, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 1, 253, 8190, 7925, 13] payload_rows_per_dst=[0, 14, 1028, 1246, 14092, 16362, 0, 26] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 14, 1028, 1246, 14092, 16362, 0, 26] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=136036352 dispatch_meta_bytes=66424 combine_payload_bytes=136036352 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=136102776 ingress_bytes=1545322496 link_bytes=1545322496 token_bytes=272139128 weight_movement_bytes=1409286144 total_nvlink_bytes=1681425272 weight_to_token_ratio=5.178550 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.653160928e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24882 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8, 0, 278, 8189, 7813, 96] payload_rows_per_dst=[496, 8172, 7886, 15466, 556, 0, 0, 192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[496, 8172, 7886, 15466, 556, 0, 0, 192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=152993792 dispatch_meta_bytes=74704 combine_payload_bytes=152993792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=153068496 ingress_bytes=1562279936 link_bytes=1562279936 token_bytes=306062288 weight_movement_bytes=1409286144 total_nvlink_bytes=1715348432 weight_to_token_ratio=4.604573 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1024 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.748115514e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32556 payload_rows_remote=16176 dup_rows=212 dup_rows_remote=0 dedup_ratio=0.006470 router_tokens_per_expert=[2, 2, 6, 0, 239, 8187, 7839, 109] payload_rows_per_dst=[4, 4, 12, 0, 478, 0, 15678, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 212] combine_rows_per_src=[4, 4, 12, 0, 478, 0, 15678, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3, 0, 238, 8191, 7905, 47] payload_rows_per_dst=[0, 8192, 7950, 16056, 476, 0, 0, 94] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 7950, 16056, 476, 0, 0, 94] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32164 payload_rows_remote=16300 dup_rows=604 dup_rows_remote=14 dedup_ratio=0.018433 router_tokens_per_expert=[0, 0, 0, 0, 261, 8192, 7924, 7] payload_rows_per_dst=[0, 0, 0, 0, 522, 22, 15864, 15756] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 590, 14] combine_rows_per_src=[0, 0, 0, 0, 522, 22, 15864, 15756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24578 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 3, 8, 0, 261, 8182, 7918, 12] payload_rows_per_dst=[16106, 8188, 7926, 0, 522, 0, 0, 24] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[16106, 8188, 7926, 0, 522, 0, 0, 24] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 258, 8192, 7901, 33] payload_rows_per_dst=[0, 0, 0, 0, 15682, 16384, 636, 66] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 15682, 16384, 636, 66] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=203833344 dispatch_meta_bytes=99528 combine_payload_bytes=203833344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=203932872 ingress_bytes=1613119488 link_bytes=1613119488 token_bytes=407766216 weight_movement_bytes=1409286144 total_nvlink_bytes=1817052360 weight_to_token_ratio=3.456113 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.032795784e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=132513792 dispatch_meta_bytes=64704 combine_payload_bytes=132513792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=132578496 ingress_bytes=1541799936 link_bytes=1541799936 token_bytes=265092288 weight_movement_bytes=1409286144 total_nvlink_bytes=1674378432 weight_to_token_ratio=5.316210 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.633436063e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=136904704 dispatch_meta_bytes=66848 combine_payload_bytes=136904704 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=136971552 ingress_bytes=1546190848 link_bytes=1546190848 token_bytes=273876256 weight_movement_bytes=1409286144 total_nvlink_bytes=1683162400 weight_to_token_ratio=5.145704 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.658023337e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=133529600 dispatch_meta_bytes=65256 combine_payload_bytes=133529600 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=133594856 ingress_bytes=5770674176 link_bytes=5770674176 token_bytes=267124456 weight_movement_bytes=5637144576 total_nvlink_bytes=5904269032 weight_to_token_ratio=21.103064 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231336659e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8188, 0, 7578, 0, 618, 0, 0] payload_rows_per_dst=[16376, 0, 0, 15156, 0, 1236, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 0, 0, 15156, 0, 1236, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98320 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441296 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784272 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070416 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=30620 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000366 router_tokens_per_expert=[0, 8186, 14, 7039, 2, 1143, 0, 0] payload_rows_per_dst=[0, 20, 16368, 0, 2136, 7316, 6916, 0] dup_rows_per_dst=[0, 0, 12, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 20, 16368, 0, 2136, 7316, 6916, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134283264 ingress_bytes=5771362304 link_bytes=5771362304 token_bytes=268500992 weight_movement_bytes=5637144576 total_nvlink_bytes=5905645568 weight_to_token_ratio=20.994874 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231721982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=250839040 dispatch_meta_bytes=122528 combine_payload_bytes=250839040 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=250961568 ingress_bytes=1660125184 link_bytes=1660125184 token_bytes=501800608 weight_movement_bytes=1409286144 total_nvlink_bytes=1911086752 weight_to_token_ratio=2.808458 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.296008060e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 25, 7487, 0, 681, 0, 0] payload_rows_per_dst=[0, 16382, 50, 0, 7726, 5158, 3452, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16382, 50, 0, 7726, 5158, 3452, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 7623, 0, 569, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8192, 1138, 8192, 15246] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 1138, 8192, 15246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8190, 1, 7672, 0, 520, 0, 1] payload_rows_per_dst=[14, 16366, 2, 2654, 6524, 7206, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[14, 16366, 2, 2654, 6524, 7206, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=17934 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[2, 8177, 39, 7414, 0, 749, 0, 3] payload_rows_per_dst=[4, 0, 78, 0, 8178, 1498, 8176, 14832] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[4, 0, 78, 0, 8178, 1498, 8176, 14832] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=25086 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000366 router_tokens_per_expert=[0, 8192, 6, 7292, 0, 894, 0, 0] payload_rows_per_dst=[0, 0, 16236, 120, 12, 7670, 6032, 2686] dup_rows_per_dst=[0, 0, 12, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16236, 120, 12, 7670, 6032, 2686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8187, 5, 7419, 0, 773, 0, 0] payload_rows_per_dst=[16374, 0, 10, 14838, 0, 1546, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16374, 0, 10, 14838, 0, 1546, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=5905580032 link_bytes=5905580032 token_bytes=537001984 weight_movement_bytes=5637144576 total_nvlink_bytes=6174146560 weight_to_token_ratio=10.497437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306878307e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16404 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8182, 8189, 2, 7, 1, 0, 0, 3] payload_rows_per_dst=[16364, 16378, 4, 14, 2, 0, 0, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16364, 16378, 4, 14, 2, 0, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 8190, 1, 1, 0, 0, 0, 1] payload_rows_per_dst=[0, 0, 16, 8178, 8190, 6, 16376, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16, 8178, 8190, 6, 16376, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=146915328 dispatch_meta_bytes=71736 combine_payload_bytes=146915328 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=146987064 ingress_bytes=1556201472 link_bytes=1556201472 token_bytes=293902392 weight_movement_bytes=1409286144 total_nvlink_bytes=1703188536 weight_to_token_ratio=4.795082 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.714078653e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=205504512 dispatch_meta_bytes=100392 combine_payload_bytes=205504512 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=205604904 ingress_bytes=1614790656 link_bytes=1614790656 token_bytes=411109416 weight_movement_bytes=1409286144 total_nvlink_bytes=1820395560 weight_to_token_ratio=3.428007 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3328 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.042153627e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=268566528 ingress_bytes=5905580032 link_bytes=5905580032 token_bytes=537001984 weight_movement_bytes=5637144576 total_nvlink_bytes=6174146560 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306878307e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134381568 dispatch_meta_bytes=65616 combine_payload_bytes=134381568 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134447184 ingress_bytes=4362240000 link_bytes=4362240000 token_bytes=268828752 weight_movement_bytes=4227858432 total_nvlink_bytes=4496687184 weight_to_token_ratio=15.726958 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442672311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 16378, 8192, 0, 8192, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16378, 8192, 0, 8192, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8192, 8192, 8, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 8192, 8, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16406 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 8190, 1, 0, 0, 2, 0, 0] payload_rows_per_dst=[22, 8, 16362, 8190, 0, 8186, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[22, 8, 16362, 8190, 0, 8186, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8184, 8188, 4, 2, 3, 0, 2, 1] payload_rows_per_dst=[0, 0, 8, 4, 8194, 8184, 4, 16374] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8, 4, 8194, 8184, 4, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 8190, 8190, 4, 16374, 10] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8190, 8190, 4, 16374, 10] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 8191, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[16382, 16382, 0, 0, 0, 4, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 0, 0, 0, 4, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8188, 1, 8189, 2, 0, 0, 3, 1] payload_rows_per_dst=[16378, 8186, 8192, 4, 0, 0, 6, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 8186, 8192, 4, 0, 0, 6, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 2, 8188, 2, 1, 0, 0, 0] payload_rows_per_dst=[0, 4, 4, 8188, 8194, 8196, 8182, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 4, 8188, 8194, 8196, 8182, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131040 combine_payload_bytes=268369920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268500960 ingress_bytes=1677656064 link_bytes=1677656064 token_bytes=536870880 weight_movement_bytes=1409286144 total_nvlink_bytes=1946157024 weight_to_token_ratio=2.625000 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394173670e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134365224 ingress_bytes=1543585792 link_bytes=1543585792 token_bytes=268664872 weight_movement_bytes=1409286144 total_nvlink_bytes=1677951016 weight_to_token_ratio=5.245517 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643436111e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134397952 dispatch_meta_bytes=65624 combine_payload_bytes=134397952 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134463576 ingress_bytes=1543684096 link_bytes=1543684096 token_bytes=268861528 weight_movement_bytes=1409286144 total_nvlink_bytes=1678147672 weight_to_token_ratio=5.241680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643986572e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=4362092544 link_bytes=4362092544 token_bytes=268533768 weight_movement_bytes=4227858432 total_nvlink_bytes=4496392200 weight_to_token_ratio=15.744234 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442589742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=134332440 ingress_bytes=5771411456 link_bytes=5771411456 token_bytes=268599320 weight_movement_bytes=5637144576 total_nvlink_bytes=5905743896 weight_to_token_ratio=20.987189 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231749505e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 8186, 8196, 8180, 2, 8204, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8186, 8196, 8180, 2, 8204, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 2, 8190, 0, 8194, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 8190, 0, 8194, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8189, 2, 8185, 3, 2, 2, 1, 0] payload_rows_per_dst=[0, 4, 0, 6, 8196, 4, 8180, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 0, 6, 8196, 4, 8180, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 1, 8191, 0, 0, 0, 1, 0] payload_rows_per_dst=[6, 8196, 8188, 8192, 0, 8184, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6, 8196, 8188, 8192, 0, 8184, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8191, 1, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 8192, 8186, 8180, 8204, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8192, 8186, 8180, 8204, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 2, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8191, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 8188, 4, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 8188, 4, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8188, 3, 8189, 0, 1, 0, 3, 0] payload_rows_per_dst=[16376, 6, 16378, 0, 2, 0, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 6, 16378, 0, 2, 0, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 1, 8191, 0, 0, 0, 1, 0] payload_rows_per_dst=[0, 2, 0, 8200, 8190, 0, 16376, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 8200, 8190, 0, 16376, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201424896 dispatch_meta_bytes=98352 combine_payload_bytes=201424896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201523248 ingress_bytes=1610711040 link_bytes=1610711040 token_bytes=402948144 weight_movement_bytes=1409286144 total_nvlink_bytes=1812234288 weight_to_token_ratio=3.497438 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019309480e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98320 combine_payload_bytes=201359360 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=201457680 ingress_bytes=5838503936 link_bytes=5838503936 token_bytes=402817040 weight_movement_bytes=5637144576 total_nvlink_bytes=6039961616 weight_to_token_ratio=13.994305 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269318493e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201424896 dispatch_meta_bytes=98352 combine_payload_bytes=201424896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201523248 ingress_bytes=1610711040 link_bytes=1610711040 token_bytes=402948144 weight_movement_bytes=1409286144 total_nvlink_bytes=1812234288 weight_to_token_ratio=3.497438 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019309480e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134348832 ingress_bytes=4362141696 link_bytes=4362141696 token_bytes=268632096 weight_movement_bytes=4227858432 total_nvlink_bytes=4496490528 weight_to_token_ratio=15.738471 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442617265e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24568 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16376, 0, 8200, 8192, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16376, 0, 8200, 8192, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8188, 4, 8188, 0, 0, 0, 4, 0] payload_rows_per_dst=[0, 8, 0, 0, 0, 16376, 8, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8, 0, 0, 0, 16376, 8, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 1, 8191, 0, 0, 0, 1, 0] payload_rows_per_dst=[8, 16376, 6, 8184, 8192, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 16376, 6, 8184, 8192, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 8184, 8192, 8, 16376, 8] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8184, 8192, 8, 16376, 8] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16388 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[8189, 0, 0, 4, 0, 8188, 2, 1] payload_rows_per_dst=[16378, 0, 8192, 8190, 0, 0, 4, 2] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 0, 8192, 8190, 0, 0, 4, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201261056 dispatch_meta_bytes=98272 combine_payload_bytes=201261056 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201359328 ingress_bytes=1610547200 link_bytes=1610547200 token_bytes=402620384 weight_movement_bytes=1409286144 total_nvlink_bytes=1811906528 weight_to_token_ratio=3.500285 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018392045e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268517352 ingress_bytes=4496244736 link_bytes=4496244736 token_bytes=536903656 weight_movement_bytes=4227858432 total_nvlink_bytes=4764762088 weight_to_token_ratio=7.874520 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517709370e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 1, 0, 0, 8191, 0, 1] payload_rows_per_dst=[0, 6, 8186, 8192, 2, 8188, 8192, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 6, 8186, 8192, 2, 8188, 8192, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131040 combine_payload_bytes=268369920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268500960 ingress_bytes=1677656064 link_bytes=1677656064 token_bytes=536870880 weight_movement_bytes=1409286144 total_nvlink_bytes=1946157024 weight_to_token_ratio=2.625000 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394173670e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134316048 ingress_bytes=5066752000 link_bytes=5066752000 token_bytes=268566544 weight_movement_bytes=4932501504 total_nvlink_bytes=5201068048 weight_to_token_ratio=18.366031 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837169623e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 16384, 0, 0, 8192, 8192, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 8192, 8192, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 1, 0, 8191, 1, 0] payload_rows_per_dst=[4, 16378, 0, 2, 8190, 8192, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 16378, 0, 2, 8190, 8192, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8188, 0, 0, 4, 0, 8188, 1, 3] payload_rows_per_dst=[0, 0, 0, 8, 8184, 0, 8194, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8, 8184, 0, 8194, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 6, 0, 8192, 4, 8186, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6, 0, 8192, 4, 8186, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 8192, 8184, 8, 8192, 8190, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 8184, 8, 8192, 8190, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3, 8186, 3, 6, 8185, 1] payload_rows_per_dst=[16372, 16370, 6, 0, 6, 12, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16372, 16370, 6, 0, 6, 12, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 0, 8192, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 8192, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 8190, 1, 3, 8188, 1] payload_rows_per_dst=[0, 0, 278, 84, 16378, 16026, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 278, 84, 16378, 16026, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201375744 dispatch_meta_bytes=98328 combine_payload_bytes=201375744 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201474072 ingress_bytes=1610661888 link_bytes=1610661888 token_bytes=402849816 weight_movement_bytes=1409286144 total_nvlink_bytes=1812135960 weight_to_token_ratio=3.498292 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019034249e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201424896 ingress_bytes=5133828096 link_bytes=5133828096 token_bytes=402751488 weight_movement_bytes=4932501504 total_nvlink_bytes=5335252992 weight_to_token_ratio=12.247010 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874729437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16412 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8186, 0, 75, 8123, 0] payload_rows_per_dst=[0, 0, 16246, 16356, 16, 150, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16246, 16356, 16, 150, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16416 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4, 8176, 4, 36, 8164, 0] payload_rows_per_dst=[0, 0, 8, 0, 8, 72, 16328, 16352] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8, 0, 8, 72, 16328, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16538 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 8183, 1, 37, 8162, 0] payload_rows_per_dst=[38, 96, 16230, 16328, 2, 74, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[38, 96, 16230, 16328, 2, 74, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8187, 0, 0, 4, 8188, 3, 2] payload_rows_per_dst=[16376, 16374, 0, 0, 8, 0, 6, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 16374, 0, 0, 8, 0, 6, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 4, 8188, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8, 16376, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16448 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 5, 8187, 0] payload_rows_per_dst=[0, 0, 0, 0, 16358, 16320, 64, 26] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16358, 16320, 64, 26] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134447104 dispatch_meta_bytes=65648 combine_payload_bytes=134447104 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134512752 ingress_bytes=5771591680 link_bytes=5771591680 token_bytes=268959856 weight_movement_bytes=5637144576 total_nvlink_bytes=5906104432 weight_to_token_ratio=20.959056 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231850423e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16466 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8179, 0, 53, 8151, 1] payload_rows_per_dst=[16358, 16302, 0, 0, 0, 106, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16358, 16302, 0, 0, 0, 106, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 1, 0, 1, 8191, 0, 0] payload_rows_per_dst=[0, 0, 8, 8204, 8180, 16376, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8, 8204, 8180, 16376, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134479872 dispatch_meta_bytes=65664 combine_payload_bytes=134479872 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134545536 ingress_bytes=1543766016 link_bytes=1543766016 token_bytes=269025408 weight_movement_bytes=1409286144 total_nvlink_bytes=1678311552 weight_to_token_ratio=5.238487 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644445290e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=135479296 dispatch_meta_bytes=66152 combine_payload_bytes=135479296 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135545448 ingress_bytes=1544765440 link_bytes=1544765440 token_bytes=271024744 weight_movement_bytes=1409286144 total_nvlink_bytes=1680310888 weight_to_token_ratio=5.199843 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.650041647e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134348832 ingress_bytes=5771427840 link_bytes=5771427840 token_bytes=268632096 weight_movement_bytes=5637144576 total_nvlink_bytes=5905776672 weight_to_token_ratio=20.984628 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231758679e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134742016 dispatch_meta_bytes=65792 combine_payload_bytes=134742016 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134807808 ingress_bytes=1544028160 link_bytes=1544028160 token_bytes=269549824 weight_movement_bytes=1409286144 total_nvlink_bytes=1678835968 weight_to_token_ratio=5.228296 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645913187e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134889472 dispatch_meta_bytes=65864 combine_payload_bytes=134889472 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134955336 ingress_bytes=1544175616 link_bytes=1544175616 token_bytes=269844808 weight_movement_bytes=1409286144 total_nvlink_bytes=1679130952 weight_to_token_ratio=5.222580 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.646738879e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201424896 dispatch_meta_bytes=98352 combine_payload_bytes=201424896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201523248 ingress_bytes=1610711040 link_bytes=1610711040 token_bytes=402948144 weight_movement_bytes=1409286144 total_nvlink_bytes=1812234288 weight_to_token_ratio=3.497438 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019309480e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24568 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 16376, 8200, 8184, 8, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16376, 8200, 8184, 8, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16392 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8185, 3, 2, 5, 8187, 1, 1] payload_rows_per_dst=[0, 0, 6, 4, 10, 0, 16372, 16374] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[0, 0, 6, 4, 10, 0, 16372, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=16384 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000183 router_tokens_per_expert=[3, 8189, 4, 3, 0, 0, 8178, 7] payload_rows_per_dst=[16378, 8192, 8, 8170, 0, 0, 0, 14] dup_rows_per_dst=[6, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 8192, 8, 8170, 0, 0, 0, 14] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201261056 dispatch_meta_bytes=98272 combine_payload_bytes=201261056 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201359328 ingress_bytes=1610547200 link_bytes=1610547200 token_bytes=402620384 weight_movement_bytes=1409286144 total_nvlink_bytes=1811906528 weight_to_token_ratio=3.500285 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018392045e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 1, 0, 1, 8191, 0, 0] payload_rows_per_dst=[8, 12, 16376, 8180, 8192, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 12, 16376, 8180, 8192, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 8180, 8192, 16376, 12, 8] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8180, 8192, 16376, 12, 8] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16388 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 2, 6, 16378, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 2, 6, 16378, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16382, 2, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16382, 2, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[41, 8191, 1, 1, 1, 1, 8147, 1] payload_rows_per_dst=[82, 0, 8192, 2, 8194, 8194, 8102, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[82, 0, 8192, 2, 8194, 8194, 8102, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=134348832 ingress_bytes=5771427840 link_bytes=5771427840 token_bytes=268632096 weight_movement_bytes=5637144576 total_nvlink_bytes=5905776672 weight_to_token_ratio=20.984628 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231758679e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65560 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316056 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566552 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852696 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=134299656 ingress_bytes=5771378688 link_bytes=5771378688 token_bytes=268533768 weight_movement_bytes=5637144576 total_nvlink_bytes=5905678344 weight_to_token_ratio=20.992312 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231731156e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8192, 0, 0, 0, 1, 8182, 8] payload_rows_per_dst=[2, 8160, 8162, 8184, 8, 8236, 0, 16] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8160, 8162, 8184, 8, 8236, 0, 16] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98336 combine_payload_bytes=201392128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201490464 ingress_bytes=1610678272 link_bytes=1610678272 token_bytes=402882592 weight_movement_bytes=1409286144 total_nvlink_bytes=1812168736 weight_to_token_ratio=3.498007 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019125993e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 0, 0, 0, 3, 3, 8187, 8188] payload_rows_per_dst=[16380, 16376, 0, 0, 6, 6, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 16376, 0, 0, 6, 6, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32732 payload_rows_remote=16364 dup_rows=36 dup_rows_remote=2 dedup_ratio=0.001099 router_tokens_per_expert=[12, 8188, 4, 4, 1, 13, 8125, 37] payload_rows_per_dst=[24, 0, 8, 8, 8146, 26, 8152, 16368] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 34] combine_rows_per_src=[24, 0, 8, 8, 8146, 26, 8152, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5, 8191, 1, 1, 0, 5, 8179, 2] payload_rows_per_dst=[10, 8190, 8194, 8236, 0, 8134, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10, 8190, 8194, 8236, 0, 8134, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=24582 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000244 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 8150, 0, 8226, 8178, 8206, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8, 0] combine_rows_per_src=[0, 0, 8150, 0, 8226, 8178, 8206, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=24468 dup_rows=64 dup_rows_remote=0 dedup_ratio=0.001953 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 8192, 0, 8236, 16276] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 64, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 0, 8236, 16276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24540 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[6, 8192, 0, 0, 0, 0, 8159, 27] payload_rows_per_dst=[16264, 8226, 54, 8168, 0, 0, 0, 54] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16264, 8226, 54, 8168, 0, 0, 0, 54] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 0, 1, 1, 8190, 8191] payload_rows_per_dst=[2, 0, 4, 10, 16378, 16374, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 4, 10, 16378, 16374, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134053888 dispatch_meta_bytes=65464 combine_payload_bytes=134053888 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134119352 ingress_bytes=1543340032 link_bytes=1543340032 token_bytes=268173240 weight_movement_bytes=1409286144 total_nvlink_bytes=1677459384 weight_to_token_ratio=5.255133 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642059957e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 2, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201375744 dispatch_meta_bytes=98360 combine_payload_bytes=201375744 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201474104 ingress_bytes=1610661888 link_bytes=1610661888 token_bytes=402849848 weight_movement_bytes=1409286144 total_nvlink_bytes=1812135992 weight_to_token_ratio=3.498291 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019034249e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=200441856 dispatch_meta_bytes=97872 combine_payload_bytes=200441856 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=200539728 ingress_bytes=5837586432 link_bytes=5837586432 token_bytes=400981584 weight_movement_bytes=5637144576 total_nvlink_bytes=6038126160 weight_to_token_ratio=14.058363 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.268804729e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 16376, 16384, 8, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16376, 16384, 8, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201031680 dispatch_meta_bytes=98168 combine_payload_bytes=201031680 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=201129848 ingress_bytes=5838176256 link_bytes=5838176256 token_bytes=402161528 weight_movement_bytes=5637144576 total_nvlink_bytes=6039306104 weight_to_token_ratio=14.017115 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269135006e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8188, 0, 4, 8185, 3, 0, 2, 2] payload_rows_per_dst=[16376, 8178, 8, 8192, 6, 0, 4, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 8178, 8, 8192, 6, 0, 4, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 0, 0, 16366, 16374, 18, 10] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16366, 16374, 18, 10] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 0, 1, 1, 8190, 8191] payload_rows_per_dst=[2, 8, 16380, 16374, 2, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8, 16380, 16374, 2, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16392 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[4, 0, 0, 0, 4, 5, 8184, 8187] payload_rows_per_dst=[8, 0, 0, 0, 8, 10, 16366, 16374] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8, 0, 0, 0, 8, 10, 16366, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 2, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 2, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 2, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16376, 16384, 8, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 16384, 8, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=24578 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[8191, 1, 1, 8191, 0, 0, 0, 0] payload_rows_per_dst=[0, 2, 8194, 16, 8186, 8192, 8174, 0] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 8194, 16, 8186, 8192, 8174, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134348832 ingress_bytes=5771427840 link_bytes=5771427840 token_bytes=268632096 weight_movement_bytes=5637144576 total_nvlink_bytes=5905776672 weight_to_token_ratio=20.984628 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231758679e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134365224 ingress_bytes=1543585792 link_bytes=1543585792 token_bytes=268664872 weight_movement_bytes=1409286144 total_nvlink_bytes=1677951016 weight_to_token_ratio=5.245517 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643436111e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134348832 ingress_bytes=5066784768 link_bytes=5066784768 token_bytes=268632096 weight_movement_bytes=4932501504 total_nvlink_bytes=5201133600 weight_to_token_ratio=18.361549 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837187972e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134283264 ingress_bytes=5066719232 link_bytes=5066719232 token_bytes=268500992 weight_movement_bytes=4932501504 total_nvlink_bytes=5201002496 weight_to_token_ratio=18.370515 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837151275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24574 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[8192, 0, 0, 8191, 0, 0, 0, 1] payload_rows_per_dst=[0, 8192, 8190, 8192, 8184, 6, 0, 2] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8190, 8192, 8184, 6, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98328 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441304 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784280 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070424 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=201408504 ingress_bytes=5838454784 link_bytes=5838454784 token_bytes=402718712 weight_movement_bytes=5637144576 total_nvlink_bytes=6039863288 weight_to_token_ratio=13.997722 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269290970e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 0, 4, 8188, 2, 3, 8185] payload_rows_per_dst=[16376, 4, 16370, 8, 0, 4, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 4, 16370, 8, 0, 4, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 8182, 4, 8192, 8186, 8198, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8182, 4, 8192, 8186, 8198, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 1, 8188, 0, 0, 1, 3] payload_rows_per_dst=[8, 8206, 8184, 8170, 8192, 0, 2, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 8206, 8184, 8170, 8192, 0, 2, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8187, 0, 5, 8182, 3, 0, 5, 2] payload_rows_per_dst=[0, 0, 10, 0, 6, 8192, 8186, 16374] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 10, 0, 6, 8192, 8186, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8192, 8204, 16372] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8192, 8204, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 8190, 0, 0, 0, 2] payload_rows_per_dst=[16384, 8190, 0, 8188, 2, 0, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8190, 0, 8188, 2, 0, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 1, 8191, 1, 1, 8190] payload_rows_per_dst=[0, 6, 0, 8198, 8184, 8186, 8194, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 6, 0, 8198, 8184, 8186, 8194, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201375744 dispatch_meta_bytes=98328 combine_payload_bytes=201375744 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201474072 ingress_bytes=1610661888 link_bytes=1610661888 token_bytes=402849816 weight_movement_bytes=1409286144 total_nvlink_bytes=1812135960 weight_to_token_ratio=3.498292 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019034249e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98336 combine_payload_bytes=201392128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201490464 ingress_bytes=1610678272 link_bytes=1610678272 token_bytes=402882592 weight_movement_bytes=1409286144 total_nvlink_bytes=1812168736 weight_to_token_ratio=3.498007 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019125993e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134365224 ingress_bytes=1543585792 link_bytes=1543585792 token_bytes=268664872 weight_movement_bytes=1409286144 total_nvlink_bytes=1677951016 weight_to_token_ratio=5.245517 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643436111e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201228288 dispatch_meta_bytes=98256 combine_payload_bytes=201228288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201326544 ingress_bytes=1610514432 link_bytes=1610514432 token_bytes=402554832 weight_movement_bytes=1409286144 total_nvlink_bytes=1811840976 weight_to_token_ratio=3.500855 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018208557e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[0, 16382, 0, 8194, 0, 8188, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16382, 0, 8194, 0, 8188, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 1, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98336 combine_payload_bytes=201392128 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=201490464 ingress_bytes=5838536704 link_bytes=5838536704 token_bytes=402882592 weight_movement_bytes=5637144576 total_nvlink_bytes=6040027168 weight_to_token_ratio=13.992028 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269336842e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7, 0, 8181, 5, 0, 8187, 2, 2] payload_rows_per_dst=[16376, 0, 6, 16378, 0, 0, 4, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 0, 6, 16378, 0, 0, 4, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 8180, 8204, 8192, 8178, 14] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8180, 8204, 8192, 8178, 14] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32754 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 1, 8191, 0, 1, 8191] payload_rows_per_dst=[8, 16374, 14, 8178, 0, 8192, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 16374, 14, 8178, 0, 8192, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 8200, 2, 8184, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8200, 2, 8184, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16396 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 1, 0, 5, 8187, 2, 5, 8184] payload_rows_per_dst=[0, 2, 0, 10, 8180, 4, 8200, 16368] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[0, 2, 0, 10, 8180, 4, 8200, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268320768 dispatch_meta_bytes=131016 combine_payload_bytes=268320768 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268451784 ingress_bytes=1677606912 link_bytes=1677606912 token_bytes=536772552 weight_movement_bytes=1409286144 total_nvlink_bytes=1946058696 weight_to_token_ratio=2.625481 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393898440e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 16348, 36, 0, 16372, 12, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16348, 36, 0, 16372, 12, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5, 1, 8186, 1, 0, 8190, 0, 1] payload_rows_per_dst=[10, 2, 16372, 2, 4, 16376, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10, 2, 16372, 2, 4, 16376, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98336 combine_payload_bytes=201392128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201490464 ingress_bytes=1610678272 link_bytes=1610678272 token_bytes=402882592 weight_movement_bytes=1409286144 total_nvlink_bytes=1812168736 weight_to_token_ratio=3.498007 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019125993e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134381616 ingress_bytes=5771460608 link_bytes=5771460608 token_bytes=268697648 weight_movement_bytes=5637144576 total_nvlink_bytes=5905842224 weight_to_token_ratio=20.979508 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231777028e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16498 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4, 0, 8114, 0, 8188, 4, 0, 74] payload_rows_per_dst=[16270, 8184, 8158, 0, 0, 8, 0, 148] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16270, 8184, 8158, 0, 0, 8, 0, 148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 16348, 0, 0, 16374, 10, 36] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16348, 0, 0, 16374, 10, 36] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1, 8189, 1, 0, 8191, 1, 0] payload_rows_per_dst=[2, 16380, 0, 4, 16380, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 16380, 0, 4, 16380, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 6, 0, 0, 6, 16378, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6, 0, 0, 6, 16378, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16420 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[17, 0, 8170, 6, 0, 8186, 1, 4] payload_rows_per_dst=[34, 0, 0, 12, 0, 0, 16374, 16348] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[34, 0, 0, 12, 0, 0, 16374, 16348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=135151616 dispatch_meta_bytes=65992 combine_payload_bytes=135151616 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135217608 ingress_bytes=1544437760 link_bytes=1544437760 token_bytes=270369224 weight_movement_bytes=1409286144 total_nvlink_bytes=1679655368 weight_to_token_ratio=5.212450 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.648206776e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16346, 38, 0, 16372, 12, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16346, 38, 0, 16372, 12, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134365224 ingress_bytes=5066801152 link_bytes=5066801152 token_bytes=268664872 weight_movement_bytes=4932501504 total_nvlink_bytes=5201166376 weight_to_token_ratio=18.359309 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837197146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=5200936960 link_bytes=5200936960 token_bytes=537001984 weight_movement_bytes=4932501504 total_nvlink_bytes=5469503488 weight_to_token_ratio=9.185258 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912307600e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8180, 0, 8192, 0, 0, 12] payload_rows_per_dst=[0, 8180, 8182, 8180, 8202, 0, 0, 24] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8180, 8182, 8180, 8202, 0, 0, 24] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 8163, 0, 8191, 1, 0, 28] payload_rows_per_dst=[2, 0, 180, 8040, 8202, 16288, 0, 56] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 180, 8040, 8202, 16288, 0, 56] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134512640 dispatch_meta_bytes=65680 combine_payload_bytes=134512640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134578320 ingress_bytes=1543798784 link_bytes=1543798784 token_bytes=269090960 weight_movement_bytes=1409286144 total_nvlink_bytes=1678377104 weight_to_token_ratio=5.237211 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644628777e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 1, 0, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268124160 dispatch_meta_bytes=130920 combine_payload_bytes=268124160 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268255080 ingress_bytes=1677410304 link_bytes=1677410304 token_bytes=536379240 weight_movement_bytes=1409286144 total_nvlink_bytes=1945665384 weight_to_token_ratio=2.627406 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.392797517e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8187, 0, 8185, 0, 1, 4, 4, 3] payload_rows_per_dst=[16374, 0, 16370, 0, 2, 8, 8, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16374, 0, 16370, 0, 2, 8, 8, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201424896 dispatch_meta_bytes=98352 combine_payload_bytes=201424896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201523248 ingress_bytes=1610711040 link_bytes=1610711040 token_bytes=402948144 weight_movement_bytes=1409286144 total_nvlink_bytes=1812234288 weight_to_token_ratio=3.497438 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019309480e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201244672 dispatch_meta_bytes=98264 combine_payload_bytes=201244672 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201342936 ingress_bytes=5133746176 link_bytes=5133746176 token_bytes=402587608 weight_movement_bytes=4932501504 total_nvlink_bytes=5335089112 weight_to_token_ratio=12.251995 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874683565e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8191, 0, 8192, 0, 0, 1] payload_rows_per_dst=[0, 0, 0, 8190, 8182, 16384, 10, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8190, 8182, 16384, 10, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134365224 ingress_bytes=4362158080 link_bytes=4362158080 token_bytes=268664872 weight_movement_bytes=4227858432 total_nvlink_bytes=4496523304 weight_to_token_ratio=15.736551 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442626439e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 8167, 0, 8191, 1, 0, 24] payload_rows_per_dst=[126, 8200, 8080, 8130, 8182, 2, 0, 48] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[126, 8200, 8080, 8130, 8182, 2, 0, 48] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[6, 0, 8102, 0, 8187, 5, 0, 84] payload_rows_per_dst=[12, 0, 0, 0, 0, 10, 16374, 16372] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12, 0, 0, 0, 0, 10, 16374, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8189, 0, 8192, 0, 0, 3] payload_rows_per_dst=[0, 0, 0, 228, 0, 76, 16384, 16080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 228, 0, 76, 16384, 16080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8173, 0, 8192, 0, 0, 19] payload_rows_per_dst=[16358, 8204, 8168, 0, 0, 0, 0, 38] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16358, 8204, 8168, 0, 0, 0, 0, 38] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4296474736 prefetch_bytes=15502147584 reduce_bytes=27481079808 weight_movement_bytes=42983227392 egress_bytes=2148761712 ingress_bytes=45130940416 link_bytes=45130940416 total_nvlink_bytes=47279702128 weight_to_token_ratio=10.004301 prefetch_share_of_ingress=0.343493 dedup_ratio=0.000843 prefetch_slots=44 unused_prefetch_slots=20 slot_utilization=0.687500 reduce_slots=39 remote_demand=22 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023674 gemm_inner_pad_upper_bound=7112 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.579465173e-04 prefetch_bytes_per_gemm_flop=5.425347222e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=202244096 dispatch_meta_bytes=98752 combine_payload_bytes=202244096 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=202342848 ingress_bytes=5134745600 link_bytes=5134745600 token_bytes=404586944 weight_movement_bytes=4932501504 total_nvlink_bytes=5337088448 weight_to_token_ratio=12.191450 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.875243201e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16384, 0, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16398 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 8191, 0, 0, 1, 1, 0] payload_rows_per_dst=[0, 12, 0, 18, 16370, 2, 16366, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 12, 0, 18, 16370, 2, 16366, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201228288 dispatch_meta_bytes=98256 combine_payload_bytes=201228288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201326544 ingress_bytes=1610514432 link_bytes=1610514432 token_bytes=402554832 weight_movement_bytes=1409286144 total_nvlink_bytes=1811840976 weight_to_token_ratio=3.500855 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018208557e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134332416 dispatch_meta_bytes=65592 combine_payload_bytes=134332416 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134398008 ingress_bytes=1543618560 link_bytes=1543618560 token_bytes=268730424 weight_movement_bytes=1409286144 total_nvlink_bytes=1678016568 weight_to_token_ratio=5.244237 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643619598e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16382, 2, 16376, 8] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16382, 2, 16376, 8] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32754 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 8190, 0, 0, 1, 1, 1] payload_rows_per_dst=[10, 16372, 14, 16366, 0, 2, 2, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10, 16372, 14, 16366, 0, 2, 2, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6319838160 prefetch_bytes=21139292160 reduce_bytes=16911433728 weight_movement_bytes=38050725888 egress_bytes=3160691664 ingress_bytes=41209872384 link_bytes=41209872384 total_nvlink_bytes=44370564048 weight_to_token_ratio=6.020839 prefetch_share_of_ingress=0.512967 dedup_ratio=0.001251 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=24 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=6858 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.442238022e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6383721272 prefetch_bytes=19730006016 reduce_bytes=10569646080 weight_movement_bytes=30299652096 egress_bytes=3192642360 ingress_bytes=33490731008 link_bytes=33490731008 total_nvlink_bytes=36683373368 weight_to_token_ratio=4.746393 prefetch_share_of_ingress=0.589118 dedup_ratio=0.002529 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=15 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019886 gemm_inner_pad_upper_bound=7366 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.172088212e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8185, 0, 8182, 0, 1, 6, 4, 6] payload_rows_per_dst=[0, 0, 0, 0, 2, 16382, 8, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 2, 16382, 8, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 12, 16372, 8, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 12, 16372, 8, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268320768 dispatch_meta_bytes=131016 combine_payload_bytes=268320768 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268451784 ingress_bytes=4496179200 link_bytes=4496179200 token_bytes=536772552 weight_movement_bytes=4227858432 total_nvlink_bytes=4764630984 weight_to_token_ratio=7.876443 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517672672e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131040 combine_payload_bytes=268369920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268500960 ingress_bytes=1677656064 link_bytes=1677656064 token_bytes=536870880 weight_movement_bytes=1409286144 total_nvlink_bytes=1946157024 weight_to_token_ratio=2.625000 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394173670e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6146090048 prefetch_bytes=17616076800 reduce_bytes=20434649088 weight_movement_bytes=38050725888 egress_bytes=3073795136 ingress_bytes=41123020800 link_bytes=41123020800 total_nvlink_bytes=44196815936 weight_to_token_ratio=6.191046 prefetch_share_of_ingress=0.428375 dedup_ratio=0.000038 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=29 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=7493 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.439198443e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6752085432 prefetch_bytes=17616076800 reduce_bytes=21843935232 weight_movement_bytes=39460012032 egress_bytes=3376866744 ingress_bytes=42835230720 link_bytes=42835230720 total_nvlink_bytes=46212097464 weight_to_token_ratio=5.844122 prefetch_share_of_ingress=0.411252 dedup_ratio=0.000000 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=31 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=7493 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.499121323e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4308208552 prefetch_bytes=21843935232 reduce_bytes=8455716864 weight_movement_bytes=30299652096 egress_bytes=2154630056 ingress_bytes=32453230592 link_bytes=32453230592 total_nvlink_bytes=34607860648 weight_to_token_ratio=7.033005 prefetch_share_of_ingress=0.673090 dedup_ratio=0.001919 prefetch_slots=62 unused_prefetch_slots=2 slot_utilization=0.968750 reduce_slots=12 remote_demand=31 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=7747 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.135778404e-04 prefetch_bytes_per_gemm_flop=7.644807449e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6078245312 prefetch_bytes=19730006016 reduce_bytes=24662507520 weight_movement_bytes=44392513536 egress_bytes=3039865280 ingress_bytes=47430893568 link_bytes=47430893568 total_nvlink_bytes=50470758848 weight_to_token_ratio=7.303508 prefetch_share_of_ingress=0.415974 dedup_ratio=0.002041 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=35 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=7493 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.659957533e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6743632664 prefetch_bytes=17616076800 reduce_bytes=20434649088 weight_movement_bytes=38050725888 egress_bytes=3372641048 ingress_bytes=41421717504 link_bytes=41421717504 total_nvlink_bytes=44794358552 weight_to_token_ratio=5.642467 prefetch_share_of_ingress=0.425286 dedup_ratio=0.001640 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=29 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023674 gemm_inner_pad_upper_bound=6731 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.449652048e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none - [2026-09-16 21:34:42] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 634.9 | throughput per GPU (TFLOP/s/GPU): 525.1 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.019930E+01 | loss scale: 1.0 | grad norm: 44.922 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=24232 dup_rows=58 dup_rows_remote=56 dedup_ratio=0.001770 router_tokens_per_expert=[3114, 4, 1521, 3183, 302, 112, 8148, 0] payload_rows_per_dst=[2370, 8, 3042, 2348, 8478, 310, 16154, 0] dup_rows_per_dst=[0, 0, 0, 0, 2, 56, 0, 0] combine_rows_per_src=[2370, 8, 3042, 2348, 8478, 310, 16154, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32754 payload_rows_remote=25936 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000427 router_tokens_per_expert=[3532, 2, 2269, 2192, 195, 11, 8181, 2] payload_rows_per_dst=[6558, 622, 6818, 2440, 926, 15386, 0, 4] dup_rows_per_dst=[4, 0, 0, 0, 0, 10, 0, 0] combine_rows_per_src=[6558, 622, 6818, 2440, 926, 15386, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2265, 4, 2211, 3308, 364, 42, 8187, 3] payload_rows_per_dst=[390, 8, 4422, 5896, 5588, 84, 16362, 18] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[390, 8, 4422, 5896, 5588, 84, 16362, 18] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16400 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[2687, 3, 1901, 3284, 264, 57, 8188, 0] payload_rows_per_dst=[8606, 16366, 3802, 3350, 528, 114, 0, 0] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8606, 16366, 3802, 3350, 528, 114, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=26834 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.002930 router_tokens_per_expert=[2637, 8, 1726, 3454, 331, 47, 8178, 3] payload_rows_per_dst=[2676, 16, 3606, 5838, 4174, 16348, 8, 6] dup_rows_per_dst=[0, 0, 0, 0, 2, 94, 0, 0] combine_rows_per_src=[2676, 16, 3606, 5838, 4174, 16348, 8, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=198508544 dispatch_meta_bytes=97152 combine_payload_bytes=198508544 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=198605696 ingress_bytes=1607794688 link_bytes=1607794688 token_bytes=397114240 weight_movement_bytes=1409286144 total_nvlink_bytes=1806400384 weight_to_token_ratio=3.548818 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1280 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.002979126e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2163, 0, 1985, 3600, 384, 62, 8189, 1] payload_rows_per_dst=[0, 0, 3726, 5666, 6628, 124, 244, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 3726, 5666, 6628, 124, 244, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=32764 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[2405, 0, 2462, 3023, 219, 96, 8176, 3] payload_rows_per_dst=[0, 0, 4924, 5324, 5968, 192, 0, 16356] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 2] combine_rows_per_src=[0, 0, 4924, 5324, 5968, 192, 0, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=212467712 dispatch_meta_bytes=103800 combine_payload_bytes=212467712 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=212571512 ingress_bytes=1621753856 link_bytes=1621753856 token_bytes=425039224 weight_movement_bytes=1409286144 total_nvlink_bytes=1834325368 weight_to_token_ratio=3.315661 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.081144640e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=267747328 dispatch_meta_bytes=130736 combine_payload_bytes=267747328 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267878064 ingress_bytes=1677033472 link_bytes=1677033472 token_bytes=535625392 weight_movement_bytes=1409286144 total_nvlink_bytes=1944911536 weight_to_token_ratio=2.631104 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.390687415e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=134348800 dispatch_meta_bytes=65600 combine_payload_bytes=134348800 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134414400 ingress_bytes=1543634944 link_bytes=1543634944 token_bytes=268763200 weight_movement_bytes=1409286144 total_nvlink_bytes=1678049344 weight_to_token_ratio=5.243598 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643711341e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32382 payload_rows_remote=20604 dup_rows=386 dup_rows_remote=0 dedup_ratio=0.011780 router_tokens_per_expert=[4691, 1, 1214, 2031, 236, 25, 8185, 1] payload_rows_per_dst=[11778, 15746, 2428, 1906, 472, 50, 0, 2] dup_rows_per_dst=[386, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[11778, 15746, 2428, 1906, 472, 50, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=219824128 dispatch_meta_bytes=107720 combine_payload_bytes=219824128 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=219931848 ingress_bytes=3743039488 link_bytes=3743039488 token_bytes=439755976 weight_movement_bytes=3523215360 total_nvlink_bytes=3962971336 weight_to_token_ratio=8.011751 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.095945871e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131072 combine_payload_bytes=268402688 prefetch_bytes=704643072 reduce_bytes=5637144576 egress_bytes=268533760 ingress_bytes=6610190336 link_bytes=6610190336 token_bytes=536936448 weight_movement_bytes=6341787648 total_nvlink_bytes=6878724096 weight_to_token_ratio=11.811058 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.701430665e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24600 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 3, 0, 0, 8192, 0, 8189] payload_rows_per_dst=[0, 0, 8196, 26, 8166, 8192, 8186, 0] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8196, 26, 8166, 8192, 8186, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=201523200 dispatch_meta_bytes=98408 combine_payload_bytes=201523200 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201621608 ingress_bytes=1610809344 link_bytes=1610809344 token_bytes=403144808 weight_movement_bytes=1409286144 total_nvlink_bytes=1812430952 weight_to_token_ratio=3.495732 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019859941e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=168787968 dispatch_meta_bytes=82416 combine_payload_bytes=168787968 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=168870384 ingress_bytes=2987360256 link_bytes=2987360256 token_bytes=337658352 weight_movement_bytes=2818572288 total_nvlink_bytes=3156230640 weight_to_token_ratio=8.347409 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.672797045e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32742 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 1, 0, 8191, 2, 8189] payload_rows_per_dst=[2, 16382, 26, 8162, 8192, 0, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 16382, 26, 8162, 8192, 0, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24568 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 4, 0, 3, 0, 8189, 0, 8187] payload_rows_per_dst=[2, 8, 8178, 6, 8184, 8200, 8190, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8, 8178, 6, 8184, 8200, 8190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 1, 0, 0, 8191, 0, 8191] payload_rows_per_dst=[16372, 12, 8194, 8188, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16372, 12, 8194, 8188, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24570 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[2, 1, 3, 5, 9, 8182, 4, 8178] payload_rows_per_dst=[4, 16364, 6, 8196, 8186, 2, 8, 0] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[4, 16364, 6, 8196, 8186, 2, 8, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=16366 dup_rows=24 dup_rows_remote=0 dedup_ratio=0.000732 router_tokens_per_expert=[2, 1, 0, 1, 0, 8188, 3, 8189] payload_rows_per_dst=[4, 2, 0, 2, 0, 8180, 8178, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 24] combine_rows_per_src=[4, 2, 0, 2, 0, 8180, 8178, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 1, 0, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=24566 dup_rows=8 dup_rows_remote=0 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 3, 8191, 5, 8185] payload_rows_per_dst=[0, 0, 0, 0, 6, 8194, 8194, 16366] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8, 0] combine_rows_per_src=[0, 0, 0, 0, 6, 8194, 8194, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 4, 0, 0, 8188, 0, 0] payload_rows_per_dst=[0, 2, 8, 0, 16382, 8184, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 8, 0, 16382, 8184, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=201261056 dispatch_meta_bytes=98272 combine_payload_bytes=201261056 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=201359328 ingress_bytes=5838405632 link_bytes=5838405632 token_bytes=402620384 weight_movement_bytes=5637144576 total_nvlink_bytes=6039764960 weight_to_token_ratio=14.001141 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269263447e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268222464 dispatch_meta_bytes=130968 combine_payload_bytes=268222464 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268353432 ingress_bytes=1677508608 link_bytes=1677508608 token_bytes=536575896 weight_movement_bytes=1409286144 total_nvlink_bytes=1945862040 weight_to_token_ratio=2.626443 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393347978e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268337152 dispatch_meta_bytes=131024 combine_payload_bytes=268337152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268468176 ingress_bytes=1677623296 link_bytes=1677623296 token_bytes=536805328 weight_movement_bytes=1409286144 total_nvlink_bytes=1946091472 weight_to_token_ratio=2.625321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393990183e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=201277440 dispatch_meta_bytes=98288 combine_payload_bytes=201277440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201375728 ingress_bytes=1610563584 link_bytes=1610563584 token_bytes=402653168 weight_movement_bytes=1409286144 total_nvlink_bytes=1811939312 weight_to_token_ratio=3.500000 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018483788e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134070272 dispatch_meta_bytes=65464 combine_payload_bytes=134070272 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134135736 ingress_bytes=5771214848 link_bytes=5771214848 token_bytes=268206008 weight_movement_bytes=5637144576 total_nvlink_bytes=5905350584 weight_to_token_ratio=21.017965 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231639413e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4, 0, 15, 8192, 0, 8173] payload_rows_per_dst=[16384, 0, 8166, 8188, 30, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 8166, 8188, 30, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=201244672 dispatch_meta_bytes=98264 combine_payload_bytes=201244672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201342936 ingress_bytes=1610530816 link_bytes=1610530816 token_bytes=402587608 weight_movement_bytes=1409286144 total_nvlink_bytes=1811873752 weight_to_token_ratio=3.500570 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018300301e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 0, 0, 8191, 0, 1] payload_rows_per_dst=[2, 8190, 8192, 16382, 0, 0, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8190, 8192, 16382, 0, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24592 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 2, 0, 0, 8189, 1, 0] payload_rows_per_dst=[2, 0, 4, 0, 16380, 8176, 8188, 18] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 4, 0, 16380, 8176, 8188, 18] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24570 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8192, 1, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16374, 8196, 8166, 30, 0, 0, 0, 0] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16374, 8196, 8166, 30, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16412 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 8189, 7, 2, 0, 8181, 2, 1] payload_rows_per_dst=[4, 8186, 8188, 16356, 4, 22, 6, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 8186, 8188, 16356, 4, 22, 6, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8, 0, 0, 8184, 0, 0] payload_rows_per_dst=[0, 0, 16, 0, 0, 8196, 8188, 16368] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16, 0, 0, 8196, 8188, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8190, 1, 0, 1, 8190, 1, 0] payload_rows_per_dst=[2, 0, 2, 0, 2, 8190, 8194, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 2, 0, 2, 8190, 8194, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24580 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 4, 8198, 8186, 16378, 0, 0] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 8198, 8186, 16378, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201277440 dispatch_meta_bytes=98280 combine_payload_bytes=201277440 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=201375720 ingress_bytes=5838422016 link_bytes=5838422016 token_bytes=402653160 weight_movement_bytes=5637144576 total_nvlink_bytes=6039797736 weight_to_token_ratio=14.000001 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269272621e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 3, 0, 0, 8189, 0, 0] payload_rows_per_dst=[16384, 8192, 8192, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201457664 dispatch_meta_bytes=98368 combine_payload_bytes=201457664 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=201556032 ingress_bytes=5838602240 link_bytes=5838602240 token_bytes=403013696 weight_movement_bytes=5637144576 total_nvlink_bytes=6040158272 weight_to_token_ratio=13.987476 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269373539e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134447104 dispatch_meta_bytes=65648 combine_payload_bytes=134447104 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134512752 ingress_bytes=1543733248 link_bytes=1543733248 token_bytes=268959856 weight_movement_bytes=1409286144 total_nvlink_bytes=1678246000 weight_to_token_ratio=5.239764 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644261803e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98320 combine_payload_bytes=201359360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201457680 ingress_bytes=1610645504 link_bytes=1610645504 token_bytes=402817040 weight_movement_bytes=1409286144 total_nvlink_bytes=1812103184 weight_to_token_ratio=3.498576 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018942506e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98328 combine_payload_bytes=201359360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201457688 ingress_bytes=1610645504 link_bytes=1610645504 token_bytes=402817048 weight_movement_bytes=1409286144 total_nvlink_bytes=1812103192 weight_to_token_ratio=3.498576 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018942506e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 1, 8191, 0, 0, 0, 1] payload_rows_per_dst=[0, 0, 16384, 8192, 8190, 0, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 8192, 8190, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8191, 0, 1, 0, 0] payload_rows_per_dst=[16376, 16384, 4, 2, 0, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 16384, 4, 2, 0, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 8187, 0, 0, 4, 1] payload_rows_per_dst=[2, 0, 0, 8186, 8192, 16378, 8, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 8186, 8192, 16378, 8, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8189, 2, 8189, 0, 0, 0, 3] payload_rows_per_dst=[2, 0, 4, 0, 0, 0, 16378, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 4, 0, 0, 0, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24580 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[3, 8189, 0, 8186, 1, 0, 2, 3] payload_rows_per_dst=[6, 0, 16372, 8186, 8192, 0, 4, 6] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[6, 0, 16372, 8186, 8192, 0, 4, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 8, 8184, 0, 8192, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8, 8184, 0, 8192, 8192, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 2, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 2, 6, 10, 16376, 16374] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 6, 10, 16376, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8191, 0, 0, 1, 0] payload_rows_per_dst=[16382, 16384, 0, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 0, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134283264 ingress_bytes=5771362304 link_bytes=5771362304 token_bytes=268500992 weight_movement_bytes=5637144576 total_nvlink_bytes=5905645568 weight_to_token_ratio=20.994874 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231721982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98328 combine_payload_bytes=201359360 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=201457688 ingress_bytes=5838503936 link_bytes=5838503936 token_bytes=402817048 weight_movement_bytes=5637144576 total_nvlink_bytes=6039961624 weight_to_token_ratio=13.994305 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269318493e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24574 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 1, 8191, 8190, 1, 0, 0, 1] payload_rows_per_dst=[0, 8190, 8192, 8192, 8190, 0, 0, 2] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8190, 8192, 8192, 8190, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8191, 8191, 1, 0, 0, 1] payload_rows_per_dst=[0, 0, 8192, 0, 8188, 8196, 8188, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 0, 8188, 8196, 8188, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8189, 8189, 3, 0, 3, 0] payload_rows_per_dst=[0, 0, 0, 0, 6, 8188, 8196, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 6, 8188, 8196, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8188, 8188, 4, 0, 4, 0] payload_rows_per_dst=[0, 8184, 8192, 8192, 8192, 0, 8, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8184, 8192, 8192, 8192, 0, 8, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 8, 0, 0, 8192, 8184, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8, 0, 0, 8192, 8184, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7903, 0, 8192, 0, 0, 289, 0, 0] payload_rows_per_dst=[0, 1888, 0, 7018, 8202, 7478, 8182, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 1888, 0, 7018, 8202, 7478, 8182, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98304 combine_payload_bytes=201310208 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201408512 ingress_bytes=4429168640 link_bytes=4429168640 token_bytes=402718720 weight_movement_bytes=4227858432 total_nvlink_bytes=4630577152 weight_to_token_ratio=10.498291 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480149556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201293824 dispatch_meta_bytes=98288 combine_payload_bytes=201293824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201392112 ingress_bytes=1610579968 link_bytes=1610579968 token_bytes=402685936 weight_movement_bytes=1409286144 total_nvlink_bytes=1811972080 weight_to_token_ratio=3.499715 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018575532e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=4429185024 link_bytes=4429185024 token_bytes=402751488 weight_movement_bytes=4227858432 total_nvlink_bytes=4630609920 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480158730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98336 combine_payload_bytes=201392128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201490464 ingress_bytes=1610678272 link_bytes=1610678272 token_bytes=402882592 weight_movement_bytes=1409286144 total_nvlink_bytes=1812168736 weight_to_token_ratio=3.498007 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019125993e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201244672 dispatch_meta_bytes=98264 combine_payload_bytes=201244672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201342936 ingress_bytes=1610530816 link_bytes=1610530816 token_bytes=402587608 weight_movement_bytes=1409286144 total_nvlink_bytes=1811873752 weight_to_token_ratio=3.500570 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018300301e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7986, 0, 8192, 0, 0, 206, 0, 0] payload_rows_per_dst=[15972, 0, 16384, 0, 0, 412, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15972, 0, 16384, 0, 0, 412, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8031, 1, 8191, 0, 1, 159, 1, 0] payload_rows_per_dst=[928, 15136, 6, 8188, 8190, 318, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[928, 15136, 6, 8188, 8190, 318, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7901, 1, 8191, 0, 0, 290, 1, 0] payload_rows_per_dst=[0, 2, 0, 7818, 8190, 8126, 8192, 440] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 7818, 8190, 8126, 8192, 440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 2, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7927, 3, 8189, 1, 0, 261, 3, 0] payload_rows_per_dst=[0, 6, 0, 2, 0, 8446, 8194, 16120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 6, 0, 2, 0, 8446, 8194, 16120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7941, 0, 8192, 0, 0, 251, 0, 0] payload_rows_per_dst=[0, 0, 0, 1546, 2, 6822, 8190, 16208] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 1546, 2, 6822, 8190, 16208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=24572 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[7864, 4, 8188, 2, 0, 322, 4, 0] payload_rows_per_dst=[0, 15736, 0, 8192, 8184, 644, 8, 0] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[0, 15736, 0, 8192, 8184, 644, 8, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16900 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[7931, 0, 8192, 0, 0, 261, 0, 0] payload_rows_per_dst=[15866, 0, 16378, 0, 0, 522, 0, 0] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15866, 0, 16378, 0, 0, 522, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 8, 16384, 16376, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8, 16384, 16376, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=268517352 ingress_bytes=5905530880 link_bytes=5905530880 token_bytes=536903656 weight_movement_bytes=5637144576 total_nvlink_bytes=6174048232 weight_to_token_ratio=10.499360 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306850784e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201867264 dispatch_meta_bytes=98568 combine_payload_bytes=201867264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201965832 ingress_bytes=1611153408 link_bytes=1611153408 token_bytes=403833096 weight_movement_bytes=1409286144 total_nvlink_bytes=1813119240 weight_to_token_ratio=3.489774 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1152 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.021786556e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=136380416 dispatch_meta_bytes=66592 combine_payload_bytes=136380416 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=136447008 ingress_bytes=1545666560 link_bytes=1545666560 token_bytes=272827424 weight_movement_bytes=1409286144 total_nvlink_bytes=1682113568 weight_to_token_ratio=5.165486 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.655087543e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201293824 dispatch_meta_bytes=98288 combine_payload_bytes=201293824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201392112 ingress_bytes=1610579968 link_bytes=1610579968 token_bytes=402685936 weight_movement_bytes=1409286144 total_nvlink_bytes=1811972080 weight_to_token_ratio=3.499715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018575532e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=138444800 dispatch_meta_bytes=67600 combine_payload_bytes=138444800 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=138512400 ingress_bytes=5775589376 link_bytes=5775589376 token_bytes=276957200 weight_movement_bytes=5637144576 total_nvlink_bytes=5914101776 weight_to_token_ratio=20.353847 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.234088966e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 8190, 8190, 4, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8190, 8190, 4, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 1, 0, 8191, 0, 0, 1] payload_rows_per_dst=[0, 8192, 8192, 16382, 0, 0, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 16382, 0, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 1, 0, 8191, 0, 0, 1] payload_rows_per_dst=[0, 0, 2, 0, 16380, 16382, 2, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 16380, 16382, 2, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 10, 16384, 16374] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 10, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8189, 0, 2, 0, 8189, 0, 1, 3] payload_rows_per_dst=[0, 0, 4, 0, 0, 0, 16380, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 0, 0, 0, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16394 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[8188, 1, 4, 1, 8186, 0, 1, 3] payload_rows_per_dst=[0, 8194, 8188, 16372, 4, 0, 2, 6] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[0, 8194, 8188, 16372, 4, 0, 2, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 8, 16376, 0, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8, 16376, 0, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 8192, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134365224 ingress_bytes=1543585792 link_bytes=1543585792 token_bytes=268664872 weight_movement_bytes=1409286144 total_nvlink_bytes=1677951016 weight_to_token_ratio=5.245517 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643436111e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16382, 16384, 2, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 2, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8188, 0, 2, 2, 1, 8191, 0, 0] payload_rows_per_dst=[0, 0, 16380, 4, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16380, 4, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8184, 0, 1, 7, 1, 8191, 0, 0] payload_rows_per_dst=[0, 0, 2, 16342, 2, 16376, 34, 12] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 16342, 2, 16376, 34, 12] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 2, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8172, 0, 3, 17, 3, 8189, 0, 0] payload_rows_per_dst=[0, 0, 6, 34, 6, 0, 16350, 16372] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6, 34, 6, 0, 16350, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8184, 1, 4, 3, 5, 8187, 0, 0] payload_rows_per_dst=[0, 2, 16370, 12, 16376, 8, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 16370, 12, 16376, 8, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 16384, 0, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 0, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134348832 ingress_bytes=5066784768 link_bytes=5066784768 token_bytes=268632096 weight_movement_bytes=4932501504 total_nvlink_bytes=5201133600 weight_to_token_ratio=18.361549 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837187972e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16382 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16382, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16382, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268337152 dispatch_meta_bytes=131024 combine_payload_bytes=268337152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268468176 ingress_bytes=1677623296 link_bytes=1677623296 token_bytes=536805328 weight_movement_bytes=1409286144 total_nvlink_bytes=1946091472 weight_to_token_ratio=2.625321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393990183e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134201344 dispatch_meta_bytes=65528 combine_payload_bytes=134201344 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134266872 ingress_bytes=5066702848 link_bytes=5066702848 token_bytes=268468216 weight_movement_bytes=4932501504 total_nvlink_bytes=5200969720 weight_to_token_ratio=18.372758 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837142100e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16368, 16, 0, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16368, 16, 0, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 2, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 0, 1, 0, 0, 8191] payload_rows_per_dst=[2, 16382, 0, 2, 16382, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 16382, 0, 2, 16382, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 0, 1, 0, 0, 8191] payload_rows_per_dst=[2, 0, 16376, 0, 2, 6, 16376, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 16376, 0, 2, 6, 16376, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268304384 dispatch_meta_bytes=131008 combine_payload_bytes=268304384 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=268435392 ingress_bytes=5200805888 link_bytes=5200805888 token_bytes=536739776 weight_movement_bytes=4932501504 total_nvlink_bytes=5469241280 weight_to_token_ratio=9.189745 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912234205e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 8189, 0, 0, 3, 0, 0, 8189] payload_rows_per_dst=[6, 0, 0, 0, 6, 16378, 0, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6, 0, 0, 0, 6, 16378, 0, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=32766 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[3, 8187, 1, 0, 5, 0, 0, 8188] payload_rows_per_dst=[6, 16368, 8, 0, 16376, 0, 8, 0] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[6, 16368, 8, 0, 16376, 0, 8, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 12, 0, 8180, 8192, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 12, 0, 8180, 8192, 8192, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268517352 ingress_bytes=1677672448 link_bytes=1677672448 token_bytes=536903656 weight_movement_bytes=1409286144 total_nvlink_bytes=1946189800 weight_to_token_ratio=2.624840 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394265414e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 2, 0, 16382, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 2, 0, 16382, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134332440 ingress_bytes=5066768384 link_bytes=5066768384 token_bytes=268599320 weight_movement_bytes=4932501504 total_nvlink_bytes=5201100824 weight_to_token_ratio=18.363790 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837178798e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131072 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550144 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969216 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255360 weight_to_token_ratio=2.624519 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 0, 16368, 4, 12, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16368, 4, 12, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 0, 1, 0, 8191, 1, 0] payload_rows_per_dst=[0, 16382, 0, 8194, 8190, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16382, 0, 8194, 8190, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 0, 1, 0, 8191, 1, 0] payload_rows_per_dst=[0, 0, 0, 8192, 8186, 8196, 8188, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8192, 8186, 8196, 8188, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8192, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8192, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8189, 3, 1, 0, 8189, 2, 0] payload_rows_per_dst=[0, 0, 6, 2, 0, 8188, 8194, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6, 2, 0, 8188, 8194, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8187, 5, 5, 0, 8187, 0, 0] payload_rows_per_dst=[0, 16374, 10, 8196, 8188, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16374, 10, 8196, 8188, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7748, 0, 0, 0, 444, 8192, 0] payload_rows_per_dst=[0, 0, 260, 4400, 16124, 11984, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 260, 4400, 16124, 11984, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201293824 dispatch_meta_bytes=98288 combine_payload_bytes=201293824 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201392112 ingress_bytes=4429152256 link_bytes=4429152256 token_bytes=402685936 weight_movement_bytes=4227858432 total_nvlink_bytes=4630544368 weight_to_token_ratio=10.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480140381e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201293824 dispatch_meta_bytes=98288 combine_payload_bytes=201293824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201392112 ingress_bytes=1610579968 link_bytes=1610579968 token_bytes=402685936 weight_movement_bytes=1409286144 total_nvlink_bytes=1811972080 weight_to_token_ratio=3.499715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018575532e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=136347648 dispatch_meta_bytes=66576 combine_payload_bytes=136347648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=136414224 ingress_bytes=1545633792 link_bytes=1545633792 token_bytes=272761872 weight_movement_bytes=1409286144 total_nvlink_bytes=1682048016 weight_to_token_ratio=5.166727 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.654904056e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32748 payload_rows_remote=17408 dup_rows=20 dup_rows_remote=0 dedup_ratio=0.000610 router_tokens_per_expert=[0, 7670, 0, 0, 0, 522, 8192, 0] payload_rows_per_dst=[16364, 15340, 0, 0, 0, 1044, 0, 0] dup_rows_per_dst=[0, 20, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16364, 15340, 0, 0, 0, 1044, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32738 payload_rows_remote=18548 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000916 router_tokens_per_expert=[1, 7328, 0, 1, 15, 846, 8191, 2] payload_rows_per_dst=[2, 0, 0, 2, 16374, 14190, 8, 2162] dup_rows_per_dst=[0, 0, 0, 0, 30, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 2, 16374, 14190, 8, 2162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32574 payload_rows_remote=16438 dup_rows=194 dup_rows_remote=194 dedup_ratio=0.005920 router_tokens_per_expert=[1, 7468, 0, 2, 15, 708, 8190, 0] payload_rows_per_dst=[2, 1018, 16136, 13972, 30, 1416, 0, 0] dup_rows_per_dst=[0, 194, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 1018, 16136, 13972, 30, 1416, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=142606336 dispatch_meta_bytes=69632 combine_payload_bytes=142606336 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=142675968 ingress_bytes=5779750912 link_bytes=5779750912 token_bytes=285282304 weight_movement_bytes=5637144576 total_nvlink_bytes=5922426880 weight_to_token_ratio=19.759882 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.236419252e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7688, 0, 0, 0, 504, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 1008, 16384, 15376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 1008, 16384, 15376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17542 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 7613, 0, 4, 34, 542, 8188, 0] payload_rows_per_dst=[6, 0, 0, 8, 68, 1084, 16376, 15226] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6, 0, 0, 8, 68, 1084, 16376, 15226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18382 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5, 7187, 0, 6, 71, 927, 8186, 2] payload_rows_per_dst=[10, 0, 16372, 14386, 142, 1854, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10, 0, 16372, 14386, 142, 1854, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16398 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8184, 8, 0] payload_rows_per_dst=[0, 14, 0, 48, 16370, 16320, 16, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 14, 0, 48, 16370, 16320, 16, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=151945216 dispatch_meta_bytes=74312 combine_payload_bytes=151945216 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=152019528 ingress_bytes=1561231360 link_bytes=1561231360 token_bytes=303964744 weight_movement_bytes=1409286144 total_nvlink_bytes=1713250888 weight_to_token_ratio=4.636347 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.742243926e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134660096 dispatch_meta_bytes=66528 combine_payload_bytes=134660096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134726624 ingress_bytes=1543946240 link_bytes=1543946240 token_bytes=269386720 weight_movement_bytes=1409286144 total_nvlink_bytes=1678672864 weight_to_token_ratio=5.231461 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645454469e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8098, 0, 0, 0, 94, 8192, 0] payload_rows_per_dst=[16384, 16196, 0, 0, 0, 188, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16196, 0, 0, 0, 188, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134283264 ingress_bytes=5771362304 link_bytes=5771362304 token_bytes=268500992 weight_movement_bytes=5637144576 total_nvlink_bytes=5905645568 weight_to_token_ratio=20.994874 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231721982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=143704064 dispatch_meta_bytes=70168 combine_payload_bytes=143704064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=143774232 ingress_bytes=1552990208 link_bytes=1552990208 token_bytes=287478296 weight_movement_bytes=1409286144 total_nvlink_bytes=1696764440 weight_to_token_ratio=4.902235 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.696096915e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=150585344 dispatch_meta_bytes=73528 combine_payload_bytes=150585344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=150658872 ingress_bytes=1559871488 link_bytes=1559871488 token_bytes=301244216 weight_movement_bytes=1409286144 total_nvlink_bytes=1710530360 weight_to_token_ratio=4.678218 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.734629210e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134332416 dispatch_meta_bytes=65592 combine_payload_bytes=134332416 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134398008 ingress_bytes=1543618560 link_bytes=1543618560 token_bytes=268730424 weight_movement_bytes=1409286144 total_nvlink_bytes=1678016568 weight_to_token_ratio=5.244237 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643619598e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8187, 5, 0] payload_rows_per_dst=[16384, 0, 16368, 6, 0, 0, 10, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16368, 6, 0, 0, 10, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 1, 0, 1, 8187, 4, 0] payload_rows_per_dst=[0, 0, 2, 0, 16378, 16374, 8, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 16378, 16374, 8, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 0, 1, 0, 2, 8191, 0, 0] payload_rows_per_dst=[0, 16380, 12, 16372, 4, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16380, 12, 16372, 4, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16458 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8175, 17, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 74, 16310, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 74, 16310, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8189, 0, 3, 0, 3, 8188, 1, 0] payload_rows_per_dst=[0, 0, 6, 0, 6, 0, 16378, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6, 0, 6, 0, 16378, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 14, 8178, 8208, 8192, 8176, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 14, 8178, 8208, 8192, 8176, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134365224 ingress_bytes=4362158080 link_bytes=4362158080 token_bytes=268664872 weight_movement_bytes=4227858432 total_nvlink_bytes=4496523304 weight_to_token_ratio=15.736551 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442626439e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16426 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8187, 0, 6, 0, 5, 8171, 15, 0] payload_rows_per_dst=[0, 16374, 12, 16342, 10, 0, 30, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16374, 12, 16342, 10, 0, 30, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268337152 dispatch_meta_bytes=131024 combine_payload_bytes=268337152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268468176 ingress_bytes=1677623296 link_bytes=1677623296 token_bytes=536805328 weight_movement_bytes=1409286144 total_nvlink_bytes=1946091472 weight_to_token_ratio=2.625321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393990183e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8184, 8, 0] payload_rows_per_dst=[16384, 0, 16368, 0, 0, 0, 16, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16368, 0, 0, 0, 16, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134823936 dispatch_meta_bytes=65832 combine_payload_bytes=134823936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134889768 ingress_bytes=1544110080 link_bytes=1544110080 token_bytes=269713704 weight_movement_bytes=1409286144 total_nvlink_bytes=1678999848 weight_to_token_ratio=5.225119 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.646371905e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201195520 dispatch_meta_bytes=98240 combine_payload_bytes=201195520 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201293760 ingress_bytes=4429053952 link_bytes=4429053952 token_bytes=402489280 weight_movement_bytes=4227858432 total_nvlink_bytes=4630347712 weight_to_token_ratio=10.504276 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480085335e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134561792 dispatch_meta_bytes=65704 combine_payload_bytes=134561792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134627496 ingress_bytes=1543847936 link_bytes=1543847936 token_bytes=269189288 weight_movement_bytes=1409286144 total_nvlink_bytes=1678475432 weight_to_token_ratio=5.235298 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644904008e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8191, 1, 8191, 0, 1, 0] payload_rows_per_dst=[0, 0, 0, 8192, 8184, 8192, 8194, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8192, 8184, 8192, 8194, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8191, 1, 8190, 0, 2, 0] payload_rows_per_dst=[0, 0, 16382, 8194, 8188, 0, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16382, 8194, 8188, 0, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8194, 8196, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8194, 8196, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8189, 4, 8188, 0, 2, 1] payload_rows_per_dst=[0, 0, 0, 8, 0, 8190, 8192, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8, 0, 8190, 8192, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 12, 0, 8192, 8192, 0, 16372, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 12, 0, 8192, 8192, 0, 16372, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8186, 6, 8186, 0, 3, 3] payload_rows_per_dst=[0, 0, 16372, 8196, 8188, 0, 6, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16372, 8196, 8188, 0, 6, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=4362092544 link_bytes=4362092544 token_bytes=268533768 weight_movement_bytes=4227858432 total_nvlink_bytes=4496392200 weight_to_token_ratio=15.744234 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442589742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201293824 dispatch_meta_bytes=98288 combine_payload_bytes=201293824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201392112 ingress_bytes=1610579968 link_bytes=1610579968 token_bytes=402685936 weight_movement_bytes=1409286144 total_nvlink_bytes=1811972080 weight_to_token_ratio=3.499715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018575532e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16380, 8194, 8192, 0, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 8194, 8192, 0, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201293824 dispatch_meta_bytes=98288 combine_payload_bytes=201293824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201392112 ingress_bytes=1610579968 link_bytes=1610579968 token_bytes=402685936 weight_movement_bytes=1409286144 total_nvlink_bytes=1811972080 weight_to_token_ratio=3.499715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018575532e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 1, 0, 8191, 1, 0] payload_rows_per_dst=[0, 0, 0, 8186, 8190, 8, 16382, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8186, 8190, 8, 16382, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 0, 0, 2, 0, 8190, 0, 2] payload_rows_per_dst=[0, 8188, 8194, 8190, 8192, 0, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8188, 8194, 8190, 8192, 0, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 2, 2, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 1, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 2, 0, 0, 0, 16384, 2, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 0, 0, 16384, 2, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131040 combine_payload_bytes=268369920 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=268500960 ingress_bytes=5905514496 link_bytes=5905514496 token_bytes=536870880 weight_movement_bytes=5637144576 total_nvlink_bytes=6174015456 weight_to_token_ratio=10.500001 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306841609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8187, 0, 0, 4, 1, 8188, 2, 2] payload_rows_per_dst=[0, 0, 0, 8, 2, 16376, 4, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8, 2, 16376, 4, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 8192, 16, 8192, 8176, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 16, 8192, 8176, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8185, 1, 0, 6, 0, 8186, 4, 2] payload_rows_per_dst=[0, 8182, 8192, 8192, 8190, 0, 8, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8182, 8192, 8192, 8190, 0, 8, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16380 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 8190, 8190, 0, 0, 0, 0, 0] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8190, 8190, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16366, 18, 8192, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16366, 18, 8192, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 8191, 0, 0, 1, 0, 8191] payload_rows_per_dst=[2, 0, 8184, 0, 8196, 8192, 8188, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 8184, 0, 8196, 8192, 8188, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134184960 dispatch_meta_bytes=65520 combine_payload_bytes=134184960 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134250480 ingress_bytes=5771329536 link_bytes=5771329536 token_bytes=268435440 weight_movement_bytes=5637144576 total_nvlink_bytes=5905580016 weight_to_token_ratio=21.000001 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231703633e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1, 8190, 0, 0, 2, 0, 8190] payload_rows_per_dst=[2, 16382, 0, 8188, 0, 8196, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 16382, 0, 8188, 0, 8196, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 8192, 0, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 0, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268288000 dispatch_meta_bytes=131000 combine_payload_bytes=268288000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268419000 ingress_bytes=1677574144 link_bytes=1677574144 token_bytes=536707000 weight_movement_bytes=1409286144 total_nvlink_bytes=1945993144 weight_to_token_ratio=2.625802 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393714952e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 1, 8188, 0, 0, 3, 0, 8189] payload_rows_per_dst=[6, 2, 0, 0, 8186, 6, 8190, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6, 2, 0, 0, 8186, 6, 8190, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24542 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 22, 8172, 8226, 16348, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 22, 8172, 8226, 16348, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4, 2, 8186, 0, 0, 6, 0, 8186] payload_rows_per_dst=[8, 16366, 8, 8180, 2, 8198, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 16366, 8, 8180, 2, 8198, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 0, 8192, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 8192, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134332440 ingress_bytes=4362125312 link_bytes=4362125312 token_bytes=268599320 weight_movement_bytes=4227858432 total_nvlink_bytes=4496457752 weight_to_token_ratio=15.740391 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442608091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201048064 dispatch_meta_bytes=98168 combine_payload_bytes=201048064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201146232 ingress_bytes=1610334208 link_bytes=1610334208 token_bytes=402194296 weight_movement_bytes=1409286144 total_nvlink_bytes=1811480440 weight_to_token_ratio=3.503993 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.017199378e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16380, 0, 4, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16380, 0, 4, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201424896 dispatch_meta_bytes=98352 combine_payload_bytes=201424896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201523248 ingress_bytes=1610711040 link_bytes=1610711040 token_bytes=402948144 weight_movement_bytes=1409286144 total_nvlink_bytes=1812234288 weight_to_token_ratio=3.497438 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019309480e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16410 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8190, 8191, 0, 0, 1, 2, 0] payload_rows_per_dst=[0, 0, 0, 8190, 8190, 16358, 4, 26] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8190, 8190, 16358, 4, 26] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6173884256 prefetch_bytes=19730006016 reduce_bytes=8455716864 weight_movement_bytes=28185722880 egress_bytes=3087695712 ingress_bytes=31271911424 link_bytes=31271911424 total_nvlink_bytes=34359607136 weight_to_token_ratio=4.565314 prefetch_share_of_ingress=0.630918 dedup_ratio=0.000118 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=12 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.021780 gemm_inner_pad_upper_bound=6858 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.094435315e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8190, 8190, 0, 0, 2, 2, 0] payload_rows_per_dst=[0, 4, 16376, 8192, 8188, 4, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 16376, 8192, 8188, 4, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16404 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 4, 16, 16364, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 4, 16, 16364, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134316048 ingress_bytes=5771395072 link_bytes=5771395072 token_bytes=268566544 weight_movement_bytes=5637144576 total_nvlink_bytes=5905711120 weight_to_token_ratio=20.989750 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231740330e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134430720 dispatch_meta_bytes=65640 combine_payload_bytes=134430720 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134496360 ingress_bytes=1543716864 link_bytes=1543716864 token_bytes=268927080 weight_movement_bytes=1409286144 total_nvlink_bytes=1678213224 weight_to_token_ratio=5.240403 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644170059e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16410 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8176, 8188, 12, 0, 3, 4, 0] payload_rows_per_dst=[2, 0, 0, 24, 0, 6, 16378, 16358] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 24, 0, 6, 16378, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134348832 ingress_bytes=5771427840 link_bytes=5771427840 token_bytes=268632096 weight_movement_bytes=5637144576 total_nvlink_bytes=5905776672 weight_to_token_ratio=20.984628 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231758679e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8175, 8185, 8, 0, 7, 9, 0] payload_rows_per_dst=[0, 0, 16370, 8186, 8158, 36, 18, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16370, 8186, 8158, 36, 18, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134381568 dispatch_meta_bytes=65616 combine_payload_bytes=134381568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134447184 ingress_bytes=1543667712 link_bytes=1543667712 token_bytes=268828752 weight_movement_bytes=1409286144 total_nvlink_bytes=1678114896 weight_to_token_ratio=5.242319 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643894829e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6460280704 prefetch_bytes=18320719872 reduce_bytes=27481079808 weight_movement_bytes=45801799680 egress_bytes=3230928768 ingress_bytes=49031151616 link_bytes=49031151616 total_nvlink_bytes=52262080384 weight_to_token_ratio=7.089754 prefetch_share_of_ingress=0.373655 dedup_ratio=0.000046 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=39 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.021307 gemm_inner_pad_upper_bound=7239 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.715962390e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6077063832 prefetch_bytes=14797504512 reduce_bytes=29595009024 weight_movement_bytes=44392513536 egress_bytes=3039273624 ingress_bytes=47430303744 link_bytes=47430303744 total_nvlink_bytes=50469577368 weight_to_token_ratio=7.304928 prefetch_share_of_ingress=0.311984 dedup_ratio=0.000057 prefetch_slots=42 unused_prefetch_slots=22 slot_utilization=0.656250 reduce_slots=42 remote_demand=21 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.024621 gemm_inner_pad_upper_bound=6985 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.659936891e-04 prefetch_bytes_per_gemm_flop=5.178740530e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134430720 dispatch_meta_bytes=65640 combine_payload_bytes=134430720 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134496360 ingress_bytes=1543716864 link_bytes=1543716864 token_bytes=268927080 weight_movement_bytes=1409286144 total_nvlink_bytes=1678213224 weight_to_token_ratio=5.240403 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644170059e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6600890584 prefetch_bytes=14797504512 reduce_bytes=24662507520 weight_movement_bytes=39460012032 egress_bytes=3301251288 ingress_bytes=42759651328 link_bytes=42759651328 total_nvlink_bytes=46060902616 weight_to_token_ratio=5.977983 prefetch_share_of_ingress=0.346062 dedup_ratio=0.000401 prefetch_slots=42 unused_prefetch_slots=22 slot_utilization=0.656250 reduce_slots=35 remote_demand=21 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.022727 gemm_inner_pad_upper_bound=7493 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.496476242e-04 prefetch_bytes_per_gemm_flop=5.178740530e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201375744 dispatch_meta_bytes=98328 combine_payload_bytes=201375744 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201474072 ingress_bytes=1610661888 link_bytes=1610661888 token_bytes=402849816 weight_movement_bytes=1409286144 total_nvlink_bytes=1812135960 weight_to_token_ratio=3.498292 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019034249e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6042845584 prefetch_bytes=20434649088 reduce_bytes=11274289152 weight_movement_bytes=31708938240 egress_bytes=3022160272 ingress_bytes=34729623552 link_bytes=34729623552 total_nvlink_bytes=37751783824 weight_to_token_ratio=5.247352 prefetch_share_of_ingress=0.588392 dedup_ratio=0.000023 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=16 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=7366 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.215446219e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4320499544 prefetch_bytes=19730006016 reduce_bytes=13388218368 weight_movement_bytes=33118224384 egress_bytes=2160777048 ingress_bytes=35277946880 link_bytes=35277946880 total_nvlink_bytes=37438723928 weight_to_token_ratio=7.665369 prefetch_share_of_ingress=0.559273 dedup_ratio=0.000046 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=19 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.022727 gemm_inner_pad_upper_bound=6604 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.234636105e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6246417712 prefetch_bytes=21139292160 reduce_bytes=10569646080 weight_movement_bytes=31708938240 egress_bytes=3123971376 ingress_bytes=34831384576 link_bytes=34831384576 total_nvlink_bytes=37955355952 weight_to_token_ratio=5.076340 prefetch_share_of_ingress=0.606904 dedup_ratio=0.000206 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=15 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019886 gemm_inner_pad_upper_bound=7112 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.219007589e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4373563888 prefetch_bytes=19025362944 reduce_bytes=22548578304 weight_movement_bytes=41573941248 egress_bytes=2187315696 ingress_bytes=43760189440 link_bytes=43760189440 total_nvlink_bytes=45947505136 weight_to_token_ratio=9.505735 prefetch_share_of_ingress=0.434764 dedup_ratio=0.000751 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=32 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=7239 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.531492465e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none - [2026-09-16 21:34:43] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 605.9 | throughput per GPU (TFLOP/s/GPU): 550.2 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.955660E+00 | loss scale: 1.0 | grad norm: 7.914 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32566 payload_rows_remote=24946 dup_rows=202 dup_rows_remote=0 dedup_ratio=0.006165 router_tokens_per_expert=[7201, 0, 4363, 4321, 447, 52, 0, 0] payload_rows_per_dst=[0, 12474, 3056, 430, 7620, 4592, 2440, 1954] dup_rows_per_dst=[0, 0, 0, 0, 202, 0, 0, 0] combine_rows_per_src=[0, 12474, 3056, 430, 7620, 4592, 2440, 1954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=22102 dup_rows=80 dup_rows_remote=80 dedup_ratio=0.002441 router_tokens_per_expert=[6284, 7, 4438, 4863, 632, 159, 1, 0] payload_rows_per_dst=[0, 14, 0, 0, 1264, 8848, 10586, 11976] dup_rows_per_dst=[0, 0, 0, 0, 0, 80, 0, 0] combine_rows_per_src=[0, 14, 0, 0, 1264, 8848, 10586, 11976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32508 payload_rows_remote=19246 dup_rows=260 dup_rows_remote=0 dedup_ratio=0.007935 router_tokens_per_expert=[6595, 3, 4387, 4869, 476, 52, 2, 0] payload_rows_per_dst=[13262, 6, 8774, 9406, 952, 104, 4, 0] dup_rows_per_dst=[260, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13262, 6, 8774, 9406, 952, 104, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=204357632 dispatch_meta_bytes=99784 combine_payload_bytes=204357632 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=204457416 ingress_bytes=1613643776 link_bytes=1613643776 token_bytes=408815048 weight_movement_bytes=1409286144 total_nvlink_bytes=1818101192 weight_to_token_ratio=3.447246 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2560 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.035731578e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=181059584 dispatch_meta_bytes=88728 combine_payload_bytes=181059584 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=181148312 ingress_bytes=1590345728 link_bytes=1590345728 token_bytes=362207896 weight_movement_bytes=1409286144 total_nvlink_bytes=1771494040 weight_to_token_ratio=3.890821 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.905272234e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32634 payload_rows_remote=28392 dup_rows=134 dup_rows_remote=134 dedup_ratio=0.004089 router_tokens_per_expert=[6817, 0, 4792, 4209, 449, 114, 0, 3] payload_rows_per_dst=[5628, 8006, 4242, 5556, 8968, 228, 0, 6] dup_rows_per_dst=[0, 0, 0, 0, 134, 0, 0, 0] combine_rows_per_src=[5628, 8006, 4242, 5556, 8968, 228, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32394 payload_rows_remote=27102 dup_rows=374 dup_rows_remote=374 dedup_ratio=0.011414 router_tokens_per_expert=[5961, 19, 4416, 5062, 817, 108, 0, 1] payload_rows_per_dst=[0, 11960, 5036, 5292, 9888, 216, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 374, 0, 0, 0] combine_rows_per_src=[0, 11960, 5036, 5292, 9888, 216, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=23564 dup_rows=8 dup_rows_remote=2 dedup_ratio=0.000244 router_tokens_per_expert=[7204, 2, 4266, 4550, 314, 47, 1, 0] payload_rows_per_dst=[0, 298, 4278, 2366, 628, 9196, 8536, 7458] dup_rows_per_dst=[0, 2, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[0, 298, 4278, 2366, 628, 9196, 8536, 7458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=32758 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[6809, 4, 4039, 4859, 576, 96, 0, 1] payload_rows_per_dst=[13618, 8, 7382, 9718, 1846, 192, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[13618, 8, 7382, 9718, 1846, 192, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32742 payload_rows_remote=21372 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000793 router_tokens_per_expert=[6362, 0, 4996, 4469, 445, 111, 1, 0] payload_rows_per_dst=[0, 0, 0, 0, 890, 9280, 11202, 11370] dup_rows_per_dst=[0, 0, 0, 0, 0, 26, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 890, 9280, 11202, 11370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=157663232 dispatch_meta_bytes=76984 combine_payload_bytes=157663232 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=157740216 ingress_bytes=5794807808 link_bytes=5794807808 token_bytes=315403448 weight_movement_bytes=5637144576 total_nvlink_bytes=5952548024 weight_to_token_ratio=17.872806 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.244850485e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 8192, 8192, 2, 16382, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 8192, 2, 16382, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=232587264 dispatch_meta_bytes=114104 combine_payload_bytes=232587264 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=232701368 ingress_bytes=3755802624 link_bytes=3755802624 token_bytes=465288632 weight_movement_bytes=3523215360 total_nvlink_bytes=3988503992 weight_to_token_ratio=7.572107 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103092695e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=222019584 dispatch_meta_bytes=109904 combine_payload_bytes=222019584 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=222129488 ingress_bytes=3745234944 link_bytes=3745234944 token_bytes=444149072 weight_movement_bytes=3523215360 total_nvlink_bytes=3967364432 weight_to_token_ratio=7.932506 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.097175235e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=193036288 dispatch_meta_bytes=94264 combine_payload_bytes=193036288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=193130552 ingress_bytes=1602322432 link_bytes=1602322432 token_bytes=386166840 weight_movement_bytes=1409286144 total_nvlink_bytes=1795452984 weight_to_token_ratio=3.649423 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.972336777e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 8190, 2, 3, 8188, 0, 0] payload_rows_per_dst=[2, 0, 0, 4, 6, 4, 16372, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 4, 6, 4, 16372, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=268353536 dispatch_meta_bytes=131040 combine_payload_bytes=268353536 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268484576 ingress_bytes=1677639680 link_bytes=1677639680 token_bytes=536838112 weight_movement_bytes=1409286144 total_nvlink_bytes=1946124256 weight_to_token_ratio=2.625160 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394081927e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=175079424 dispatch_meta_bytes=85592 combine_payload_bytes=175079424 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=175165016 ingress_bytes=1584365568 link_bytes=1584365568 token_bytes=350244440 weight_movement_bytes=1409286144 total_nvlink_bytes=1759530584 weight_to_token_ratio=4.023722 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.871785834e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 3, 8192, 0, 0, 8189, 0, 0] payload_rows_per_dst=[16384, 8198, 2, 8184, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8198, 2, 8184, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 8192, 8192, 0, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 0, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 8192, 8192, 8212, 8172, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8192, 8192, 8212, 8172, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32758 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 0, 8185, 5, 0, 8188, 3, 0] payload_rows_per_dst=[6, 8176, 8194, 10, 16368, 8, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6, 8176, 8194, 10, 16368, 8, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 0, 8190, 2, 0, 8190, 0, 0] payload_rows_per_dst=[4, 0, 8188, 8192, 0, 16374, 6, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 8188, 8192, 0, 16374, 6, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24564 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[1, 0, 8192, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16370, 8202, 0, 8186, 8, 0, 0, 0] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16370, 8202, 0, 8186, 8, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=5133828096 link_bytes=5133828096 token_bytes=402751488 weight_movement_bytes=4932501504 total_nvlink_bytes=5335252992 weight_to_token_ratio=12.247010 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874729437e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 3, 8189, 1, 0, 8189, 1, 0] payload_rows_per_dst=[2, 6, 0, 2, 8190, 0, 8194, 16374] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 6, 0, 2, 8190, 0, 8194, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268353536 dispatch_meta_bytes=131032 combine_payload_bytes=268353536 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268484568 ingress_bytes=1677639680 link_bytes=1677639680 token_bytes=536838104 weight_movement_bytes=1409286144 total_nvlink_bytes=1946124248 weight_to_token_ratio=2.625160 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394081927e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134365224 ingress_bytes=5066801152 link_bytes=5066801152 token_bytes=268664872 weight_movement_bytes=4932501504 total_nvlink_bytes=5201166376 weight_to_token_ratio=18.359309 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837197146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=201228288 dispatch_meta_bytes=98264 combine_payload_bytes=201228288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201326552 ingress_bytes=1610514432 link_bytes=1610514432 token_bytes=402554840 weight_movement_bytes=1409286144 total_nvlink_bytes=1811840984 weight_to_token_ratio=3.500855 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018208557e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16436 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 18, 54, 16330, 16364, 0, 0] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 18, 54, 16330, 16364, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 4, 16384, 8192, 0, 8188, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 16384, 8192, 0, 8188, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 2, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[10, 1, 8182, 0, 0, 8183, 8, 0] payload_rows_per_dst=[20, 2, 16358, 8188, 4, 8180, 16, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[20, 2, 16358, 8188, 4, 8180, 16, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134643712 dispatch_meta_bytes=65752 combine_payload_bytes=134643712 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134709464 ingress_bytes=1543929856 link_bytes=1543929856 token_bytes=269353176 weight_movement_bytes=1409286144 total_nvlink_bytes=1678639320 weight_to_token_ratio=5.232113 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645362726e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16440 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 1, 8183, 0, 0, 8, 0] payload_rows_per_dst=[2, 0, 2, 0, 54, 24, 16328, 16358] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 2, 0, 54, 24, 16328, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=134283264 ingress_bytes=5771362304 link_bytes=5771362304 token_bytes=268500992 weight_movement_bytes=5637144576 total_nvlink_bytes=5905645568 weight_to_token_ratio=20.994874 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231721982e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 0, 8190, 1, 0, 8190, 1, 0] payload_rows_per_dst=[4, 0, 0, 8188, 8192, 8186, 8194, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 0, 8188, 8192, 8186, 8194, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8191, 1, 0, 8189, 0, 3] payload_rows_per_dst=[16358, 16372, 26, 6, 0, 0, 0, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16358, 16372, 26, 6, 0, 0, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8186, 0, 0, 6, 0] payload_rows_per_dst=[16372, 16384, 0, 0, 0, 0, 12, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16372, 16384, 0, 0, 0, 0, 12, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8190, 2, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8190, 2, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98320 combine_payload_bytes=201359360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201457680 ingress_bytes=1610645504 link_bytes=1610645504 token_bytes=402817040 weight_movement_bytes=1409286144 total_nvlink_bytes=1812103184 weight_to_token_ratio=3.498576 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018942506e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134676480 dispatch_meta_bytes=65760 combine_payload_bytes=134676480 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134742240 ingress_bytes=1543962624 link_bytes=1543962624 token_bytes=269418720 weight_movement_bytes=1409286144 total_nvlink_bytes=1678704864 weight_to_token_ratio=5.230840 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645546213e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201375744 dispatch_meta_bytes=98328 combine_payload_bytes=201375744 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=201474072 ingress_bytes=5838520320 link_bytes=5838520320 token_bytes=402849816 weight_movement_bytes=5637144576 total_nvlink_bytes=6039994392 weight_to_token_ratio=13.993167 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269327667e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1249, 6943, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 2498, 13886, 0, 0, 28, 16356, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2498, 13886, 0, 0, 28, 16356, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[14, 0, 16384, 16370, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[14, 0, 16384, 16370, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1885, 6306, 8191, 0, 0, 1, 0] payload_rows_per_dst=[2, 3770, 0, 0, 6506, 6106, 18, 16366] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 3770, 0, 0, 6506, 6106, 18, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16428 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8, 8178, 3, 8171, 0, 0, 13, 11] payload_rows_per_dst=[16, 0, 16362, 16340, 2, 0, 26, 22] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16, 0, 16362, 16340, 2, 0, 26, 22] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8190, 1, 8188, 0, 0, 2, 2] payload_rows_per_dst=[2, 0, 2, 0, 16376, 16380, 4, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 2, 0, 16376, 16380, 4, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18368 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 992, 7200, 8192, 0, 0, 0, 0] payload_rows_per_dst=[14400, 1984, 6, 16378, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[14400, 1984, 6, 16378, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8182, 3, 0, 7, 0] payload_rows_per_dst=[16362, 16384, 0, 2, 6, 0, 14, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16362, 16384, 0, 2, 6, 0, 14, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268288000 dispatch_meta_bytes=131000 combine_payload_bytes=268288000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268419000 ingress_bytes=1677574144 link_bytes=1677574144 token_bytes=536707000 weight_movement_bytes=1409286144 total_nvlink_bytes=1945993144 weight_to_token_ratio=2.625802 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393714952e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8191, 0, 0, 1, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134578176 dispatch_meta_bytes=65712 combine_payload_bytes=134578176 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134643888 ingress_bytes=5067079680 link_bytes=5067079680 token_bytes=269222064 weight_movement_bytes=4932501504 total_nvlink_bytes=5201723568 weight_to_token_ratio=18.321312 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837353110e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 8192, 0, 8192, 26, 16358, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 0, 8192, 26, 16358, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=150470656 dispatch_meta_bytes=73472 combine_payload_bytes=150470656 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=150544128 ingress_bytes=1559756800 link_bytes=1559756800 token_bytes=301014784 weight_movement_bytes=1409286144 total_nvlink_bytes=1710300928 weight_to_token_ratio=4.681784 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.733987005e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134283264 ingress_bytes=5066719232 link_bytes=5066719232 token_bytes=268500992 weight_movement_bytes=4932501504 total_nvlink_bytes=5201002496 weight_to_token_ratio=18.370515 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837151275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32538 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1449, 6743, 8192, 0, 0, 0, 0] payload_rows_per_dst=[4386, 11768, 230, 8, 8192, 8184, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4386, 11768, 230, 8, 8192, 8184, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 8191, 3, 0, 8188, 1, 0] payload_rows_per_dst=[0, 2, 0, 6, 8190, 8186, 2, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 6, 8190, 8186, 2, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 1615, 6565, 8181, 6, 0, 6, 8] payload_rows_per_dst=[6, 3230, 13130, 0, 8204, 8170, 12, 16] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6, 3230, 13130, 0, 8204, 8170, 12, 16] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=266551296 dispatch_meta_bytes=130152 combine_payload_bytes=266551296 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=266681448 ingress_bytes=5199052800 link_bytes=5199052800 token_bytes=533232744 weight_movement_bytes=4932501504 total_nvlink_bytes=5465734248 weight_to_token_ratio=9.250185 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.911252549e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28118 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1437, 6753, 8190, 1, 0, 1, 1] payload_rows_per_dst=[2, 2874, 5516, 0, 3342, 4650, 16382, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 2874, 5516, 0, 3342, 4650, 16382, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=268566528 ingress_bytes=5200936960 link_bytes=5200936960 token_bytes=537001984 weight_movement_bytes=4932501504 total_nvlink_bytes=5469503488 weight_to_token_ratio=9.185258 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912307600e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30358 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1205, 6986, 8191, 1, 1, 0, 0] payload_rows_per_dst=[13972, 2410, 0, 16382, 2, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13972, 2410, 0, 16382, 2, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2117, 6075, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 4234, 0, 0, 6522, 5628, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4234, 0, 0, 6522, 5628, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 26, 16358, 0, 0, 30, 16354, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 26, 16358, 0, 0, 30, 16354, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=230342656 dispatch_meta_bytes=112472 combine_payload_bytes=230342656 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=230455128 ingress_bytes=1639628800 link_bytes=1639628800 token_bytes=460797784 weight_movement_bytes=1409286144 total_nvlink_bytes=1870083928 weight_to_token_ratio=3.058361 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.181236865e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=248692736 dispatch_meta_bytes=121432 combine_payload_bytes=248692736 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=248814168 ingress_bytes=1657978880 link_bytes=1657978880 token_bytes=497506904 weight_movement_bytes=1409286144 total_nvlink_bytes=1906793048 weight_to_token_ratio=2.832697 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.283989654e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[2, 8190, 8192, 8194, 0, 8190, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8190, 8192, 8194, 0, 8190, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 0, 8191, 0, 1, 1] payload_rows_per_dst=[0, 0, 0, 8190, 0, 8192, 28, 16358] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8190, 0, 8192, 28, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=268566528 ingress_bytes=5200936960 link_bytes=5200936960 token_bytes=537001984 weight_movement_bytes=4932501504 total_nvlink_bytes=5469503488 weight_to_token_ratio=9.185258 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912307600e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 12, 8180, 0, 2, 8178, 12, 0] payload_rows_per_dst=[0, 8188, 8196, 8184, 12, 8164, 24, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8188, 8196, 8184, 12, 8164, 24, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16382 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 0, 0, 8192, 8190, 0, 0, 0] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 0, 8192, 8190, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 8190, 0, 0, 8190, 2, 0] payload_rows_per_dst=[0, 4, 8188, 0, 8186, 6, 16382, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 8188, 0, 8186, 6, 16382, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201424896 ingress_bytes=5133828096 link_bytes=5133828096 token_bytes=402751488 weight_movement_bytes=4932501504 total_nvlink_bytes=5335252992 weight_to_token_ratio=12.247010 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874729437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268206080 dispatch_meta_bytes=130960 combine_payload_bytes=268206080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268337040 ingress_bytes=1677492224 link_bytes=1677492224 token_bytes=536543120 weight_movement_bytes=1409286144 total_nvlink_bytes=1945829264 weight_to_token_ratio=2.626604 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393256235e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98336 combine_payload_bytes=201392128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201490464 ingress_bytes=1610678272 link_bytes=1610678272 token_bytes=402882592 weight_movement_bytes=1409286144 total_nvlink_bytes=1812168736 weight_to_token_ratio=3.498007 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019125993e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 8191, 4, 0, 8187, 1, 0] payload_rows_per_dst=[16382, 8192, 0, 8192, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8192, 0, 8192, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134201344 dispatch_meta_bytes=65528 combine_payload_bytes=134201344 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134266872 ingress_bytes=5066702848 link_bytes=5066702848 token_bytes=268468216 weight_movement_bytes=4932501504 total_nvlink_bytes=5200969720 weight_to_token_ratio=18.372758 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837142100e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8188, 8196, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8188, 8196, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=268517352 ingress_bytes=5200887808 link_bytes=5200887808 token_bytes=536903656 weight_movement_bytes=4932501504 total_nvlink_bytes=5469405160 weight_to_token_ratio=9.186940 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912280077e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 8220, 8164, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8220, 8164, 8192, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 16384, 0, 0, 8196, 8188, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 8196, 8188, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 8191, 1, 0, 8191, 0] payload_rows_per_dst=[0, 0, 2, 0, 2, 8192, 8190, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 2, 8192, 8190, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=201555968 dispatch_meta_bytes=98416 combine_payload_bytes=201555968 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201654384 ingress_bytes=1610842112 link_bytes=1610842112 token_bytes=403210352 weight_movement_bytes=1409286144 total_nvlink_bytes=1812496496 weight_to_token_ratio=3.495164 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.020043429e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8178, 1, 12, 1, 8179, 0, 2, 11] payload_rows_per_dst=[0, 16358, 24, 2, 8192, 8166, 4, 22] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16358, 24, 2, 8192, 8166, 4, 22] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 0, 2, 0, 8190, 0, 0, 2] payload_rows_per_dst=[0, 0, 16384, 0, 0, 0, 16380, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 0, 0, 0, 16380, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201441288 ingress_bytes=4429201408 link_bytes=4429201408 token_bytes=402784264 weight_movement_bytes=4227858432 total_nvlink_bytes=4630642696 weight_to_token_ratio=10.496583 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480167905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 1, 0, 8191, 0, 1, 0] payload_rows_per_dst=[16382, 0, 2, 8192, 8190, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 0, 2, 8192, 8190, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24600 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 20, 0, 8216, 8168, 8172, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 20, 0, 8216, 8168, 8172, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 8192, 0, 8192, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8192, 0, 8192, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[2, 0, 16384, 8190, 8192, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 16384, 8190, 8192, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 0, 0, 1, 1, 0, 8191] payload_rows_per_dst=[0, 0, 0, 2, 8192, 2, 8190, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 8192, 2, 8190, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201523200 dispatch_meta_bytes=98400 combine_payload_bytes=201523200 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201621600 ingress_bytes=1610809344 link_bytes=1610809344 token_bytes=403144800 weight_movement_bytes=1409286144 total_nvlink_bytes=1812430944 weight_to_token_ratio=3.495732 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019859941e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24600 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 12, 8179, 13, 0, 8178, 0] payload_rows_per_dst=[0, 4, 16374, 8168, 8216, 4, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 16374, 8168, 8216, 4, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 8190, 2, 0, 8190, 0] payload_rows_per_dst=[0, 0, 4, 8190, 8192, 8188, 8192, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 8190, 8192, 8188, 8192, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=201523200 dispatch_meta_bytes=98400 combine_payload_bytes=201523200 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201621600 ingress_bytes=4429381632 link_bytes=4429381632 token_bytes=403144800 weight_movement_bytes=4227858432 total_nvlink_bytes=4631003232 weight_to_token_ratio=10.487196 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480268822e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 8191, 1, 0, 8191, 0] payload_rows_per_dst=[16382, 16380, 4, 0, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16380, 4, 0, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[0, 0, 0, 16384, 28, 0, 16356, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 28, 0, 16356, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98320 combine_payload_bytes=201359360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201457680 ingress_bytes=1610645504 link_bytes=1610645504 token_bytes=402817040 weight_movement_bytes=1409286144 total_nvlink_bytes=1812103184 weight_to_token_ratio=3.498576 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018942506e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8192, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8192, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 16384, 2, 8190, 0, 8192, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 2, 8190, 0, 8192, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268206080 dispatch_meta_bytes=130960 combine_payload_bytes=268206080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268337040 ingress_bytes=1677492224 link_bytes=1677492224 token_bytes=536543120 weight_movement_bytes=1409286144 total_nvlink_bytes=1945829264 weight_to_token_ratio=2.626604 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393256235e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 0, 8191, 1, 8191] payload_rows_per_dst=[0, 0, 2, 0, 0, 16382, 2, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 0, 16382, 2, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=24598 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[4, 8178, 0, 1, 11, 10, 0, 8180] payload_rows_per_dst=[8, 16356, 0, 8168, 22, 8210, 2, 0] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[8, 16356, 0, 8168, 22, 8210, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8190, 0, 0, 2, 2, 0, 8190] payload_rows_per_dst=[0, 0, 0, 8190, 8192, 8192, 8192, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8190, 8192, 8192, 8192, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201506816 dispatch_meta_bytes=98392 combine_payload_bytes=201506816 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201605208 ingress_bytes=1610792960 link_bytes=1610792960 token_bytes=403112024 weight_movement_bytes=1409286144 total_nvlink_bytes=1812398168 weight_to_token_ratio=3.496016 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019768198e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 0, 1, 0, 0, 8191] payload_rows_per_dst=[16376, 8, 16382, 0, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16376, 8, 16382, 0, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32738 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 40, 16344, 30, 8192, 8162, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 40, 16344, 30, 8192, 8162, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 8192, 0, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 0, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[2, 2, 16382, 0, 16382, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 2, 16382, 0, 16382, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131040 combine_payload_bytes=268369920 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268500960 ingress_bytes=4496228352 link_bytes=4496228352 token_bytes=536870880 weight_movement_bytes=4227858432 total_nvlink_bytes=4764729312 weight_to_token_ratio=7.875000 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517700195e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268189696 dispatch_meta_bytes=130952 combine_payload_bytes=268189696 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268320648 ingress_bytes=4496048128 link_bytes=4496048128 token_bytes=536510344 weight_movement_bytes=4227858432 total_nvlink_bytes=4764368776 weight_to_token_ratio=7.880292 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517599277e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 1, 8177, 0, 14, 0] payload_rows_per_dst=[2, 0, 0, 6, 0, 8188, 8190, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 6, 0, 8188, 8190, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 13, 0, 0, 8179, 13, 8179] payload_rows_per_dst=[0, 0, 16378, 6, 16358, 0, 26, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16378, 6, 16358, 0, 26, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16404 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8182, 0, 10, 0] payload_rows_per_dst=[16364, 16384, 0, 0, 0, 0, 20, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16364, 16384, 0, 0, 0, 0, 20, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 0, 0, 8190, 2, 8190] payload_rows_per_dst=[0, 0, 4, 16378, 0, 2, 16382, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 16378, 0, 2, 16382, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268517352 ingress_bytes=1677672448 link_bytes=1677672448 token_bytes=536903656 weight_movement_bytes=1409286144 total_nvlink_bytes=1946189800 weight_to_token_ratio=2.624840 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394265414e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 0, 8191, 1, 8191] payload_rows_per_dst=[16382, 16382, 2, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 2, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16408 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[0, 24, 0, 20, 16360, 0, 16364, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 24, 0, 20, 16360, 0, 16364, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134381568 dispatch_meta_bytes=65616 combine_payload_bytes=134381568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134447184 ingress_bytes=1543667712 link_bytes=1543667712 token_bytes=268828752 weight_movement_bytes=1409286144 total_nvlink_bytes=1678114896 weight_to_token_ratio=5.242319 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643894829e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268550136 ingress_bytes=4496277504 link_bytes=4496277504 token_bytes=536969208 weight_movement_bytes=4227858432 total_nvlink_bytes=4764827640 weight_to_token_ratio=7.873558 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517727718e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8191, 0, 1, 0] payload_rows_per_dst=[4, 2, 16382, 0, 16378, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 2, 16382, 0, 16378, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 1, 0, 0, 8191, 0, 8191] payload_rows_per_dst=[2, 0, 2, 0, 0, 16382, 0, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 2, 0, 0, 16382, 0, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134414336 dispatch_meta_bytes=65632 combine_payload_bytes=134414336 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134479968 ingress_bytes=1543700480 link_bytes=1543700480 token_bytes=268894304 weight_movement_bytes=1409286144 total_nvlink_bytes=1678180448 weight_to_token_ratio=5.241041 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644078316e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[13, 8173, 0, 17, 8179, 2, 0, 0] payload_rows_per_dst=[26, 0, 16346, 34, 16358, 4, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[26, 0, 16346, 34, 16358, 4, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 2, 8190] payload_rows_per_dst=[16384, 8192, 8188, 0, 0, 0, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 8188, 0, 0, 0, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 8190, 0, 2, 8190, 0, 0, 0] payload_rows_per_dst=[4, 0, 0, 16384, 0, 8190, 8188, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 0, 16384, 0, 8190, 8188, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268156928 dispatch_meta_bytes=130936 combine_payload_bytes=268156928 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268287864 ingress_bytes=1677443072 link_bytes=1677443072 token_bytes=536444792 weight_movement_bytes=1409286144 total_nvlink_bytes=1945730936 weight_to_token_ratio=2.627085 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.392981004e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 0, 8184, 1, 7, 0] payload_rows_per_dst=[16368, 16382, 0, 0, 2, 2, 14, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16368, 16382, 0, 0, 2, 2, 14, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 26, 8166, 8192, 0, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 26, 8166, 8192, 0, 8192, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8191, 0, 1, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8192, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8192, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[0, 8192, 8192, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8191, 1, 0, 8191, 0, 1] payload_rows_per_dst=[0, 0, 28, 2, 8192, 2, 8190, 16354] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 28, 2, 8192, 2, 8190, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=4362092544 link_bytes=4362092544 token_bytes=268533768 weight_movement_bytes=4227858432 total_nvlink_bytes=4496392200 weight_to_token_ratio=15.744234 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442589742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16422 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 14, 0, 13, 8179, 4, 8174] payload_rows_per_dst=[0, 8166, 8218, 16346, 28, 0, 10, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8166, 8218, 16346, 28, 0, 10, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 2, 1, 0, 8190, 2, 8188] payload_rows_per_dst=[2, 0, 4, 2, 16378, 2, 16378, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 4, 2, 16378, 2, 16378, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134529024 dispatch_meta_bytes=65688 combine_payload_bytes=134529024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134594712 ingress_bytes=1543815168 link_bytes=1543815168 token_bytes=269123736 weight_movement_bytes=1409286144 total_nvlink_bytes=1678409880 weight_to_token_ratio=5.236573 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644720521e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 1, 8191, 6, 8185] payload_rows_per_dst=[16380, 8194, 8164, 16, 2, 0, 12, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 8194, 8164, 16, 2, 0, 12, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268550136 ingress_bytes=4496277504 link_bytes=4496277504 token_bytes=536969208 weight_movement_bytes=4227858432 total_nvlink_bytes=4764827640 weight_to_token_ratio=7.873558 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517727718e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 6, 16378, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 6, 16378, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[2, 8190, 8192, 0, 8192, 8192, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8190, 8192, 0, 8192, 8192, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 8191, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 2, 2, 0, 2, 16382, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 2, 0, 2, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=32736 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[0, 2, 8178, 14, 0, 8176, 2, 12] payload_rows_per_dst=[0, 8164, 8192, 28, 8160, 8192, 4, 24] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8164, 8192, 28, 8160, 8192, 4, 24] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=4429185024 link_bytes=4429185024 token_bytes=402751488 weight_movement_bytes=4227858432 total_nvlink_bytes=4630609920 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480158730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8190, 2, 0, 8190, 0, 2] payload_rows_per_dst=[0, 0, 8190, 8194, 0, 8190, 8190, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8190, 8194, 0, 8190, 8190, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268173312 dispatch_meta_bytes=130960 combine_payload_bytes=268173312 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268304272 ingress_bytes=1677459456 link_bytes=1677459456 token_bytes=536477584 weight_movement_bytes=1409286144 total_nvlink_bytes=1945763728 weight_to_token_ratio=2.626925 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393072748e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8191, 1, 0, 8191, 0, 1] payload_rows_per_dst=[16382, 8192, 0, 8160, 32, 0, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8192, 0, 8160, 32, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 16384, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201441288 ingress_bytes=4429201408 link_bytes=4429201408 token_bytes=402784264 weight_movement_bytes=4227858432 total_nvlink_bytes=4630642696 weight_to_token_ratio=10.496583 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480167905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 8191, 0, 1, 0, 8191] payload_rows_per_dst=[2, 0, 0, 0, 16382, 2, 0, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 0, 16382, 2, 0, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2, 2, 16382, 16382, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 2, 16382, 16382, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8192, 0, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 0, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8178, 8170, 12, 17, 5, 0, 0, 2] payload_rows_per_dst=[0, 0, 16378, 16372, 12, 2, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16378, 16372, 12, 2, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 8190, 2, 2, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 4, 4, 16378, 16380, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 4, 16378, 16380, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 8191, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16382, 2, 2, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 2, 2, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24536 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 736, 7456, 8232, 16344, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 736, 7456, 8232, 16344, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16420 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 8163, 0, 0, 0, 1, 28] payload_rows_per_dst=[2, 0, 0, 800, 60, 36, 16348, 15522] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 800, 60, 36, 16348, 15522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 16384, 2, 0, 0, 16382, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 2, 0, 0, 16382, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=4362092544 link_bytes=4362092544 token_bytes=268533768 weight_movement_bytes=4227858432 total_nvlink_bytes=4496392200 weight_to_token_ratio=15.744234 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442589742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=200998912 dispatch_meta_bytes=98144 combine_payload_bytes=200998912 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201097056 ingress_bytes=1610285056 link_bytes=1610285056 token_bytes=402095968 weight_movement_bytes=1409286144 total_nvlink_bytes=1811382112 weight_to_token_ratio=3.504850 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.016924148e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[12, 6, 0, 8174, 0, 14, 0, 8178] payload_rows_per_dst=[24, 16354, 0, 6, 0, 16378, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[24, 16354, 0, 6, 0, 16378, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134512640 dispatch_meta_bytes=65680 combine_payload_bytes=134512640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134578320 ingress_bytes=1543798784 link_bytes=1543798784 token_bytes=269090960 weight_movement_bytes=1409286144 total_nvlink_bytes=1678377104 weight_to_token_ratio=5.237211 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644628777e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1, 0, 8190, 0, 2, 0, 8190] payload_rows_per_dst=[2, 2, 0, 16378, 2, 4, 16378, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 2, 0, 16378, 2, 4, 16378, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6987646552 prefetch_bytes=19730006016 reduce_bytes=9160359936 weight_movement_bytes=28890365952 egress_bytes=3494676056 ingress_bytes=32383336448 link_bytes=32383336448 total_nvlink_bytes=35878012504 weight_to_token_ratio=4.134492 prefetch_share_of_ingress=0.609264 dedup_ratio=0.000389 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=13 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.024148 gemm_inner_pad_upper_bound=6223 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.133332291e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 8191, 0, 1, 0, 8191] payload_rows_per_dst=[16356, 28, 16382, 0, 0, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16356, 28, 16382, 0, 0, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6672997264 prefetch_bytes=21139292160 reduce_bytes=4227858432 weight_movement_bytes=25367150592 egress_bytes=3337313168 ingress_bytes=28702834688 link_bytes=28702834688 total_nvlink_bytes=32040147856 weight_to_token_ratio=3.801463 prefetch_share_of_ingress=0.736488 dedup_ratio=0.000153 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=6 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020360 gemm_inner_pad_upper_bound=6985 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.004524332e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268517352 ingress_bytes=4496244736 link_bytes=4496244736 token_bytes=536903656 weight_movement_bytes=4227858432 total_nvlink_bytes=4764762088 weight_to_token_ratio=7.874520 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517709370e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268206080 dispatch_meta_bytes=130960 combine_payload_bytes=268206080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268337040 ingress_bytes=1677492224 link_bytes=1677492224 token_bytes=536543120 weight_movement_bytes=1409286144 total_nvlink_bytes=1945829264 weight_to_token_ratio=2.626604 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393256235e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[12, 2, 16376, 8192, 8186, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12, 2, 16376, 8192, 8186, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4375923760 prefetch_bytes=19730006016 reduce_bytes=13388218368 weight_movement_bytes=33118224384 egress_bytes=2188495920 ingress_bytes=35305652224 link_bytes=35305652224 total_nvlink_bytes=37494148144 weight_to_token_ratio=7.568282 prefetch_share_of_ingress=0.558834 dedup_ratio=0.000500 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=19 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.022254 gemm_inner_pad_upper_bound=6731 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.235605720e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[13, 8175, 7828, 0, 0, 4, 15, 349] payload_rows_per_dst=[26, 0, 15654, 8192, 8160, 8, 30, 698] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[26, 0, 15654, 8192, 8160, 8, 30, 698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134348832 ingress_bytes=5066784768 link_bytes=5066784768 token_bytes=268632096 weight_movement_bytes=4932501504 total_nvlink_bytes=5201133600 weight_to_token_ratio=18.361549 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837187972e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 8190, 8127, 0, 0, 0, 2, 63] payload_rows_per_dst=[4, 0, 0, 8124, 8130, 16380, 4, 126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 0, 8124, 8130, 16380, 4, 126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 8170, 2, 0, 0, 1, 19] payload_rows_per_dst=[16340, 16382, 2, 4, 0, 0, 2, 38] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16340, 16382, 2, 4, 0, 0, 2, 38] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6100204104 prefetch_bytes=16911433728 reduce_bytes=28890365952 weight_movement_bytes=45801799680 egress_bytes=3050846792 ingress_bytes=48851156992 link_bytes=48851156992 total_nvlink_bytes=51902003784 weight_to_token_ratio=7.508241 prefetch_share_of_ingress=0.346183 dedup_ratio=0.000256 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=41 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.022254 gemm_inner_pad_upper_bound=7239 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.709663048e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134299656 ingress_bytes=5066735616 link_bytes=5066735616 token_bytes=268533768 weight_movement_bytes=4932501504 total_nvlink_bytes=5201035272 weight_to_token_ratio=18.368273 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837160449e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7023439448 prefetch_bytes=16911433728 reduce_bytes=20434649088 weight_movement_bytes=37346082816 egress_bytes=3512577624 ingress_bytes=40856944640 link_bytes=40856944640 total_nvlink_bytes=44369522264 weight_to_token_ratio=5.317350 prefetch_share_of_ingress=0.413918 dedup_ratio=0.000725 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=29 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.021780 gemm_inner_pad_upper_bound=7366 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.429886472e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6620096488 prefetch_bytes=15502147584 reduce_bytes=26776436736 weight_movement_bytes=42278584320 egress_bytes=3310856168 ingress_bytes=45587824640 link_bytes=45587824640 total_nvlink_bytes=48898680808 weight_to_token_ratio=6.386400 prefetch_share_of_ingress=0.340050 dedup_ratio=0.000015 prefetch_slots=44 unused_prefetch_slots=20 slot_utilization=0.687500 reduce_slots=38 remote_demand=22 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.025095 gemm_inner_pad_upper_bound=6731 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.595454929e-04 prefetch_bytes_per_gemm_flop=5.425347222e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6135437784 prefetch_bytes=15502147584 reduce_bytes=22548578304 weight_movement_bytes=38050725888 egress_bytes=3068467672 ingress_bytes=41117696000 link_bytes=41117696000 total_nvlink_bytes=44186163672 weight_to_token_ratio=6.201795 prefetch_share_of_ingress=0.377019 dedup_ratio=0.000008 prefetch_slots=44 unused_prefetch_slots=20 slot_utilization=0.687500 reduce_slots=32 remote_demand=22 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023674 gemm_inner_pad_upper_bound=7112 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.439012089e-04 prefetch_bytes_per_gemm_flop=5.425347222e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4377759320 prefetch_bytes=16911433728 reduce_bytes=16911433728 weight_movement_bytes=33822867456 egress_bytes=2189413976 ingress_bytes=36011212800 link_bytes=36011212800 total_nvlink_bytes=38200626776 weight_to_token_ratio=7.726068 prefetch_share_of_ingress=0.469616 dedup_ratio=0.000050 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=24 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.024148 gemm_inner_pad_upper_bound=6731 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.260298499e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none - [2026-09-16 21:34:43] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 584.5 | throughput per GPU (TFLOP/s/GPU): 570.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.059309E+00 | loss scale: 1.0 | grad norm: 6.584 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3882, 1, 1529, 2194, 6188, 2589, 0, 1] payload_rows_per_dst=[1928, 5838, 3058, 4388, 0, 5178, 12376, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[1928, 5838, 3058, 4388, 0, 5178, 12376, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3338, 15, 1287, 1915, 7184, 2645, 0, 0] payload_rows_per_dst=[0, 6706, 2574, 3382, 448, 5290, 0, 14368] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 6706, 2574, 3382, 448, 5290, 0, 14368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31572 payload_rows_remote=23660 dup_rows=1196 dup_rows_remote=816 dedup_ratio=0.036499 router_tokens_per_expert=[2862, 9, 692, 2610, 6970, 3237, 3, 1] payload_rows_per_dst=[7912, 18, 14508, 4948, 4178, 0, 6, 2] dup_rows_per_dst=[380, 0, 816, 0, 0, 0, 0, 0] combine_rows_per_src=[7912, 18, 14508, 4948, 4178, 0, 6, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=19094 dup_rows=58 dup_rows_remote=56 dedup_ratio=0.001770 router_tokens_per_expert=[4478, 10, 1391, 1749, 6803, 1947, 0, 6] payload_rows_per_dst=[0, 8102, 2782, 1288, 2210, 4712, 0, 13616] dup_rows_per_dst=[0, 4, 0, 0, 0, 52, 0, 2] combine_rows_per_src=[0, 8102, 2782, 1288, 2210, 4712, 0, 13616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=31162 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000244 router_tokens_per_expert=[4325, 4, 799, 2300, 6685, 2268, 2, 1] payload_rows_per_dst=[8650, 8, 1598, 4696, 13370, 4432, 4, 2] dup_rows_per_dst=[0, 0, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[8650, 8, 1598, 4696, 13370, 4432, 4, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=5905580032 link_bytes=5905580032 token_bytes=537001984 weight_movement_bytes=5637144576 total_nvlink_bytes=6174146560 weight_to_token_ratio=10.497437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306878307e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27832 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4479, 0, 868, 2468, 6662, 1907, 0, 0] payload_rows_per_dst=[4682, 4276, 1736, 4936, 1316, 3814, 12008, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4682, 4276, 1736, 4936, 1316, 3814, 12008, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=26128 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000183 router_tokens_per_expert=[3896, 11, 771, 1814, 6574, 3317, 0, 1] payload_rows_per_dst=[0, 7808, 1542, 3628, 0, 6634, 8374, 4776] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 7808, 1542, 3628, 0, 6634, 8374, 4776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=193822720 dispatch_meta_bytes=97904 combine_payload_bytes=193822720 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=193920624 ingress_bytes=3717038080 link_bytes=3717038080 token_bytes=387743344 weight_movement_bytes=3523215360 total_nvlink_bytes=3910958704 weight_to_token_ratio=9.086463 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.081386168e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=156418048 dispatch_meta_bytes=76600 combine_payload_bytes=156418048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=156494648 ingress_bytes=1565704192 link_bytes=1565704192 token_bytes=312912696 weight_movement_bytes=1409286144 total_nvlink_bytes=1722198840 weight_to_token_ratio=4.503768 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.767289918e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=255279104 dispatch_meta_bytes=124680 combine_payload_bytes=255279104 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=255403784 ingress_bytes=1664565248 link_bytes=1664565248 token_bytes=510682888 weight_movement_bytes=1409286144 total_nvlink_bytes=1919969032 weight_to_token_ratio=2.759611 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4224 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.320870565e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=22784 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7343, 8192, 0, 0, 849, 0, 0] payload_rows_per_dst=[0, 0, 4, 16380, 9984, 6400, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 16380, 9984, 6400, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=227999744 dispatch_meta_bytes=111328 combine_payload_bytes=227999744 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=228111072 ingress_bytes=2341928960 link_bytes=2341928960 token_bytes=456110816 weight_movement_bytes=2113929216 total_nvlink_bytes=2570040032 weight_to_token_ratio=4.634683 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.311382461e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=214040576 dispatch_meta_bytes=104536 combine_payload_bytes=214040576 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=214145112 ingress_bytes=3032612864 link_bytes=3032612864 token_bytes=428185688 weight_movement_bytes=2818572288 total_nvlink_bytes=3246757976 weight_to_token_ratio=6.582593 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.698136616e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19440 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 6664, 8184, 0, 0, 1530, 0, 6] payload_rows_per_dst=[0, 0, 0, 4, 0, 3060, 13328, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 4, 0, 3060, 13328, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32350 payload_rows_remote=32348 dup_rows=418 dup_rows_remote=418 dedup_ratio=0.012756 router_tokens_per_expert=[4164, 1, 367, 2436, 7304, 2112, 0, 0] payload_rows_per_dst=[8944, 2, 4066, 5436, 11246, 2656, 0, 0] dup_rows_per_dst=[272, 0, 88, 58, 0, 0, 0, 0] combine_rows_per_src=[8944, 2, 4066, 5436, 11246, 2656, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=186646528 dispatch_meta_bytes=91136 combine_payload_bytes=186646528 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=186737664 ingress_bytes=1595932672 link_bytes=1595932672 token_bytes=373384192 weight_movement_bytes=1409286144 total_nvlink_bytes=1782670336 weight_to_token_ratio=3.774359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.936556788e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 6393, 8191, 0, 0, 1797, 0, 3] payload_rows_per_dst=[16382, 12786, 0, 0, 0, 3594, 0, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 12786, 0, 0, 0, 3594, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=159252480 dispatch_meta_bytes=77760 combine_payload_bytes=159252480 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=159330240 ingress_bytes=1568538624 link_bytes=1568538624 token_bytes=318582720 weight_movement_bytes=1409286144 total_nvlink_bytes=1727868864 weight_to_token_ratio=4.423611 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.783161554e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7149, 8192, 0, 0, 1043, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 2086, 14298, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 2086, 14298, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16388 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[2, 7250, 8189, 0, 0, 940, 3, 0] payload_rows_per_dst=[4, 6622, 16376, 0, 7876, 1880, 6, 0] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 6622, 16376, 0, 7876, 1880, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=264994816 dispatch_meta_bytes=131064 combine_payload_bytes=264994816 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=265125880 ingress_bytes=1674280960 link_bytes=1674280960 token_bytes=530120696 weight_movement_bytes=1409286144 total_nvlink_bytes=1939406840 weight_to_token_ratio=2.658425 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.375274497e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7454, 8192, 0, 0, 738, 0, 0] payload_rows_per_dst=[0, 0, 16382, 2, 14908, 1476, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16382, 2, 14908, 1476, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 8194, 8190, 0, 2, 16382, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8194, 8190, 0, 2, 16382, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=21520 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 6399, 8192, 0, 0, 1793, 0, 0] payload_rows_per_dst=[0, 0, 0, 16382, 0, 11248, 5136, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16382, 0, 11248, 5136, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=134316048 ingress_bytes=5771395072 link_bytes=5771395072 token_bytes=268566544 weight_movement_bytes=5637144576 total_nvlink_bytes=5905711120 weight_to_token_ratio=20.989750 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231740330e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=32766 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 8191, 1, 1, 8189, 0, 2] payload_rows_per_dst=[0, 0, 0, 2, 16382, 0, 0, 16382] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2, 16382, 0, 0, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=176291840 dispatch_meta_bytes=86080 combine_payload_bytes=176291840 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=176377920 ingress_bytes=1585577984 link_bytes=1585577984 token_bytes=352669760 weight_movement_bytes=1409286144 total_nvlink_bytes=1761955904 weight_to_token_ratio=3.996050 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5376 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.878574858e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131072 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550144 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969216 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255360 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16382, 2, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16382, 2, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19408 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 6680, 8192, 0, 0, 1512, 0, 0] payload_rows_per_dst=[16382, 13360, 2, 0, 0, 3024, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 13360, 2, 0, 0, 3024, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 8190, 1, 0, 8190, 2, 0] payload_rows_per_dst=[2, 8190, 8190, 2, 0, 16380, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 8190, 8190, 2, 0, 16380, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16410 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8190, 0, 2, 8192, 0] payload_rows_per_dst=[0, 0, 0, 26, 16358, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 26, 16358, 8192, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 8192, 8192, 0, 0, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 0, 0, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 2, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 8192, 8190, 2, 0, 16382, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 8190, 2, 0, 16382, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 1, 8153, 0, 38, 8191, 0] payload_rows_per_dst=[0, 2, 2, 0, 36, 8232, 8190, 16306] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 2, 0, 36, 8232, 8190, 16306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201441288 ingress_bytes=5133844480 link_bytes=5133844480 token_bytes=402784264 weight_movement_bytes=4932501504 total_nvlink_bytes=5335285768 weight_to_token_ratio=12.246013 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874738612e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134430720 dispatch_meta_bytes=65640 combine_payload_bytes=134430720 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134496360 ingress_bytes=1543716864 link_bytes=1543716864 token_bytes=268927080 weight_movement_bytes=1409286144 total_nvlink_bytes=1678213224 weight_to_token_ratio=5.240403 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644170059e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=158990336 dispatch_meta_bytes=77632 combine_payload_bytes=158990336 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=159067968 ingress_bytes=5796134912 link_bytes=5796134912 token_bytes=318058304 weight_movement_bytes=5637144576 total_nvlink_bytes=5955202880 weight_to_token_ratio=17.723620 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.245593608e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16406 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8180, 0, 12, 8192, 0] payload_rows_per_dst=[16362, 16382, 0, 0, 0, 24, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16362, 16382, 0, 0, 0, 24, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=5200936960 link_bytes=5200936960 token_bytes=537001984 weight_movement_bytes=4932501504 total_nvlink_bytes=5469503488 weight_to_token_ratio=9.185258 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912307600e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201441288 ingress_bytes=5133844480 link_bytes=5133844480 token_bytes=402784264 weight_movement_bytes=4932501504 total_nvlink_bytes=5335285768 weight_to_token_ratio=12.246013 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874738612e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8192, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8192, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134397952 dispatch_meta_bytes=65624 combine_payload_bytes=134397952 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134463576 ingress_bytes=1543684096 link_bytes=1543684096 token_bytes=268861528 weight_movement_bytes=1409286144 total_nvlink_bytes=1678147672 weight_to_token_ratio=5.241680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643986572e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 4, 0, 4, 16380, 8190, 8190, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 0, 4, 16380, 8190, 8190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 0, 2, 8188, 0, 2, 8190, 0] payload_rows_per_dst=[24, 0, 16384, 16356, 0, 4, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[24, 0, 16384, 16356, 0, 4, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8191, 0, 0, 0, 1] payload_rows_per_dst=[0, 0, 0, 0, 2, 8194, 8192, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 2, 8194, 8192, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16382 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 16380, 16384, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16380, 16384, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8183, 0, 9, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 16372, 8124, 8194, 78] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16372, 8124, 8194, 78] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16382, 8194, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8194, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134201344 dispatch_meta_bytes=65528 combine_payload_bytes=134201344 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134266872 ingress_bytes=5066702848 link_bytes=5066702848 token_bytes=268468216 weight_movement_bytes=4932501504 total_nvlink_bytes=5200969720 weight_to_token_ratio=18.372758 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837142100e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201883648 dispatch_meta_bytes=98576 combine_payload_bytes=201883648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201982224 ingress_bytes=1611169792 link_bytes=1611169792 token_bytes=403865872 weight_movement_bytes=1409286144 total_nvlink_bytes=1813152016 weight_to_token_ratio=3.489491 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.021878300e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8192, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8192, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 6934, 0, 8192, 1258, 0, 0] payload_rows_per_dst=[0, 8, 5272, 8184, 8192, 4232, 6880, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8, 5272, 8184, 8192, 4232, 6880, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8190, 0, 8190, 1, 0, 1, 2] payload_rows_per_dst=[0, 16380, 0, 16380, 2, 0, 2, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16380, 0, 16380, 2, 0, 2, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 1, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25358 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 7347, 15, 8177, 844, 0, 0] payload_rows_per_dst=[2, 0, 0, 30, 8192, 1688, 7410, 15446] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 30, 8192, 1688, 7410, 15446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 16384, 0, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 8192, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18090 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 6754, 5, 8187, 1438, 0, 0] payload_rows_per_dst=[14678, 8190, 7014, 10, 0, 2876, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[14678, 8190, 7014, 10, 0, 2876, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201424896 ingress_bytes=5133828096 link_bytes=5133828096 token_bytes=402751488 weight_movement_bytes=4932501504 total_nvlink_bytes=5335252992 weight_to_token_ratio=12.247010 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874729437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16382, 16384, 2, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 2, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=207732736 dispatch_meta_bytes=101432 combine_payload_bytes=207732736 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=207834168 ingress_bytes=1617018880 link_bytes=1617018880 token_bytes=415566904 weight_movement_bytes=1409286144 total_nvlink_bytes=1824853048 weight_to_token_ratio=3.391238 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.054630751e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=148193280 dispatch_meta_bytes=72360 combine_payload_bytes=148193280 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=148265640 ingress_bytes=1557479424 link_bytes=1557479424 token_bytes=296458920 weight_movement_bytes=1409286144 total_nvlink_bytes=1705745064 weight_to_token_ratio=4.753732 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.721234650e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16546 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 7352, 0, 8192, 840, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8192, 1680, 6674, 16222] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 1680, 6674, 16222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 2, 8192, 2, 8192, 8190, 8190, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 8192, 2, 8192, 8190, 8190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26352 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1, 6654, 1, 8190, 1535, 2, 0] payload_rows_per_dst=[2324, 8186, 6416, 8190, 0, 7648, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2324, 8186, 6416, 8190, 0, 7648, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=135544832 dispatch_meta_bytes=66184 combine_payload_bytes=135544832 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135611016 ingress_bytes=1544830976 link_bytes=1544830976 token_bytes=271155848 weight_movement_bytes=1409286144 total_nvlink_bytes=1680441992 weight_to_token_ratio=5.197329 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.650408621e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 2, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 0, 0, 8192, 0, 0, 1] payload_rows_per_dst=[0, 0, 0, 0, 2, 8194, 8192, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 2, 8194, 8192, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 7169, 0, 8192, 1023, 0, 0] payload_rows_per_dst=[0, 8192, 6710, 8162, 30, 6250, 3424, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 6710, 8162, 30, 6250, 3424, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=4429185024 link_bytes=4429185024 token_bytes=402751488 weight_movement_bytes=4227858432 total_nvlink_bytes=4630609920 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480158730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25830 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 7163, 0, 8192, 1029, 0, 0] payload_rows_per_dst=[0, 0, 0, 8192, 8162, 6938, 8376, 1100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8192, 8162, 6938, 8376, 1100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=215875584 dispatch_meta_bytes=105408 combine_payload_bytes=215875584 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=215980992 ingress_bytes=5148377088 link_bytes=5148377088 token_bytes=431856576 weight_movement_bytes=4932501504 total_nvlink_bytes=5364358080 weight_to_token_ratio=11.421620 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.882876265e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 8192, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 8192, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201572352 dispatch_meta_bytes=98424 combine_payload_bytes=201572352 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201670776 ingress_bytes=1610858496 link_bytes=1610858496 token_bytes=403243128 weight_movement_bytes=1409286144 total_nvlink_bytes=1812529272 weight_to_token_ratio=3.494880 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.020135172e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=211599360 dispatch_meta_bytes=103320 combine_payload_bytes=211599360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=211702680 ingress_bytes=1620885504 link_bytes=1620885504 token_bytes=423302040 weight_movement_bytes=1409286144 total_nvlink_bytes=1832588184 weight_to_token_ratio=3.329268 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4992 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.076282231e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 16384, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8190, 1, 0, 8190, 0, 1, 2] payload_rows_per_dst=[0, 8190, 8192, 8190, 8190, 0, 2, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8190, 8192, 8190, 8190, 0, 2, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 8192, 8192, 8192, 8192, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 8192, 8192, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 8192, 0, 8192, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 0, 8192, 8192, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 7464, 0, 8192, 728, 0, 0] payload_rows_per_dst=[15764, 8192, 7356, 0, 0, 1456, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15764, 8192, 7356, 0, 0, 1456, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 4, 0, 16380, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 0, 16380, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 8190, 2, 0, 8190, 0] payload_rows_per_dst=[0, 0, 16384, 0, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 0, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 4, 16380, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 4, 16380, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 16384, 0, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 0, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 16384, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 2, 8190, 8192, 0, 4, 16380, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 8190, 8192, 0, 4, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8190, 0, 0, 2, 2, 0, 8190] payload_rows_per_dst=[0, 16380, 0, 0, 4, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16380, 0, 0, 4, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=4429185024 link_bytes=4429185024 token_bytes=402751488 weight_movement_bytes=4227858432 total_nvlink_bytes=4630609920 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480158730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 0, 1, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 16384, 0, 0, 0, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 0, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 16384, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24562 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 1, 8191] payload_rows_per_dst=[0, 0, 8194, 8, 8206, 0, 16360, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8194, 8, 8206, 0, 16360, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 0, 2, 0, 0, 8190, 2, 0] payload_rows_per_dst=[0, 8190, 8194, 0, 0, 16380, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8190, 8194, 0, 0, 16380, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 21, 8171] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 46, 16338] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 46, 16338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=201211904 dispatch_meta_bytes=98248 combine_payload_bytes=201211904 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201310152 ingress_bytes=1610498048 link_bytes=1610498048 token_bytes=402522056 weight_movement_bytes=1409286144 total_nvlink_bytes=1811808200 weight_to_token_ratio=3.501140 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018116814e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 8192, 8192, 0, 0, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 0, 0, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268058624 dispatch_meta_bytes=130888 combine_payload_bytes=268058624 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268189512 ingress_bytes=1677344768 link_bytes=1677344768 token_bytes=536248136 weight_movement_bytes=1409286144 total_nvlink_bytes=1945534280 weight_to_token_ratio=2.628049 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.392430543e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 7, 8185] payload_rows_per_dst=[16382, 8192, 0, 8180, 0, 0, 14, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8192, 0, 8180, 0, 0, 14, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[0, 0, 8192, 8192, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 8192, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16380 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 16380, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[0, 0, 0, 0, 0, 16380, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 8190, 0, 0, 2, 2, 3, 8185] payload_rows_per_dst=[4, 8190, 8190, 8196, 8178, 4, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 8190, 8190, 8196, 8178, 4, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134184960 dispatch_meta_bytes=65520 combine_payload_bytes=134184960 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134250480 ingress_bytes=5066686464 link_bytes=5066686464 token_bytes=268435440 weight_movement_bytes=4932501504 total_nvlink_bytes=5200936944 weight_to_token_ratio=18.375001 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837132926e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 8192, 8192, 8192, 8192, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 8192, 8192, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 2, 8190] payload_rows_per_dst=[0, 0, 8192, 0, 8192, 0, 16342, 42] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 0, 8192, 0, 16342, 42] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17122 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 38, 8154, 0, 0] payload_rows_per_dst=[0, 0, 4, 738, 15646, 0, 16380, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 738, 15646, 0, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 0, 2, 0, 0, 8190, 8190, 0] payload_rows_per_dst=[4, 16380, 4, 16380, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 16380, 4, 16380, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 110, 8082, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 220, 16164, 4, 16380] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 220, 16164, 4, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 16384, 0, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=140263424 dispatch_meta_bytes=68488 combine_payload_bytes=140263424 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=140331912 ingress_bytes=1549549568 link_bytes=1549549568 token_bytes=280595336 weight_movement_bytes=1409286144 total_nvlink_bytes=1689881480 weight_to_token_ratio=5.022486 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.676830767e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 165, 8027, 0, 0] payload_rows_per_dst=[16384, 16054, 0, 0, 330, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16054, 0, 0, 330, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 131, 8061, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 262, 16122, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 262, 16122, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16382, 8194, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8194, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 2, 8190, 0, 8192, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 8190, 0, 8192, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8190, 2, 133, 8057, 0, 2] payload_rows_per_dst=[0, 412, 16380, 15706, 266, 0, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 412, 16380, 15706, 266, 0, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 1, 0, 0, 8191, 0, 0, 0] payload_rows_per_dst=[0, 2, 0, 0, 0, 0, 16382, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 0, 0, 0, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=201408504 ingress_bytes=5133811712 link_bytes=5133811712 token_bytes=402718712 weight_movement_bytes=4932501504 total_nvlink_bytes=5335220216 weight_to_token_ratio=12.248007 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.874720263e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=4429185024 link_bytes=4429185024 token_bytes=402751488 weight_movement_bytes=4227858432 total_nvlink_bytes=4630609920 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480158730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16444 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 30, 8162, 0, 0] payload_rows_per_dst=[0, 0, 16384, 16324, 60, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 16324, 60, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134316048 ingress_bytes=4362108928 link_bytes=4362108928 token_bytes=268566544 weight_movement_bytes=4227858432 total_nvlink_bytes=4496424976 weight_to_token_ratio=15.742312 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442598916e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134709248 dispatch_meta_bytes=65776 combine_payload_bytes=134709248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134775024 ingress_bytes=1543995392 link_bytes=1543995392 token_bytes=269484272 weight_movement_bytes=1409286144 total_nvlink_bytes=1678770416 weight_to_token_ratio=5.229567 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645729700e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32286 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 137, 8055, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 15902, 482, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 15902, 482, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16380, 4, 16380, 4, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 4, 16380, 4, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3601, 4591, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 3352, 8194, 4, 16380, 4838, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 3352, 8194, 4, 16380, 4838, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=264486912 dispatch_meta_bytes=129144 combine_payload_bytes=264486912 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=264616056 ingress_bytes=4492345344 link_bytes=4492345344 token_bytes=529102968 weight_movement_bytes=4227858432 total_nvlink_bytes=4756961400 weight_to_token_ratio=7.990616 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.515525873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 0, 2, 2, 8190, 0, 0, 0] payload_rows_per_dst=[0, 8190, 8194, 8194, 8190, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8190, 8194, 8194, 8190, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=22926 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3178, 5013, 0, 0, 1, 8192] payload_rows_per_dst=[0, 0, 0, 6542, 0, 0, 9842, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 6542, 0, 0, 9842, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 8192, 8192, 8190, 8194, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 8190, 8194, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3838, 4354, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 8708, 7676, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8708, 7676, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=187809792 dispatch_meta_bytes=91704 combine_payload_bytes=187809792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=187901496 ingress_bytes=1597095936 link_bytes=1597095936 token_bytes=375711288 weight_movement_bytes=1409286144 total_nvlink_bytes=1784997432 weight_to_token_ratio=3.750982 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.943070581e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2236, 5956, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 4776, 0, 0, 11608, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 4776, 0, 0, 11608, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 8192, 0, 8190, 16384, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 0, 8190, 16384, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16466 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 41, 8151, 0, 0] payload_rows_per_dst=[16384, 16302, 0, 0, 82, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16302, 0, 0, 82, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 8194, 2, 8190, 8192, 8190, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8194, 2, 8190, 8192, 8190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27028 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 2605, 5580, 0, 2, 5, 8190] payload_rows_per_dst=[0, 10634, 5740, 0, 16380, 4, 10, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 10634, 5740, 0, 16380, 4, 10, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 7, 0, 0, 8185] payload_rows_per_dst=[0, 0, 0, 0, 8206, 2, 8190, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8206, 2, 8190, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134889472 dispatch_meta_bytes=65864 combine_payload_bytes=134889472 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134955336 ingress_bytes=1544175616 link_bytes=1544175616 token_bytes=269844808 weight_movement_bytes=1409286144 total_nvlink_bytes=1679130952 weight_to_token_ratio=5.222580 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.646738879e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27790 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2806, 5386, 0, 0, 0, 8192] payload_rows_per_dst=[0, 812, 9614, 4978, 16384, 0, 980, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 812, 9614, 4978, 16384, 0, 980, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=221413376 dispatch_meta_bytes=108112 combine_payload_bytes=221413376 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=221521488 ingress_bytes=2335342592 link_bytes=2335342592 token_bytes=442934864 weight_movement_bytes=2113929216 total_nvlink_bytes=2556864080 weight_to_token_ratio=4.772551 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.307694370e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 1, 0, 0, 8191] payload_rows_per_dst=[16382, 8192, 0, 8192, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8192, 0, 8192, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 8192, 0, 8192, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 0, 8192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6695481280 prefetch_bytes=18320719872 reduce_bytes=17616076800 weight_movement_bytes=35936796672 egress_bytes=3348557760 ingress_bytes=39283720192 link_bytes=39283720192 total_nvlink_bytes=42632277952 weight_to_token_ratio=5.367321 prefetch_share_of_ingress=0.466369 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=25 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023674 gemm_inner_pad_upper_bound=6604 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.374827720e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3132, 5060, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 2616, 8278, 0, 16384, 5490, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2616, 8278, 0, 16384, 5490, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=227655680 dispatch_meta_bytes=111160 combine_payload_bytes=227655680 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=227766840 ingress_bytes=3750871040 link_bytes=3750871040 token_bytes=455422520 weight_movement_bytes=3523215360 total_nvlink_bytes=3978637880 weight_to_token_ratio=7.736147 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.100331213e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7284629888 prefetch_bytes=18320719872 reduce_bytes=13388218368 weight_movement_bytes=31708938240 egress_bytes=3643203968 ingress_bytes=35350364160 link_bytes=35350364160 total_nvlink_bytes=38993568128 weight_to_token_ratio=4.352855 prefetch_share_of_ingress=0.518261 dedup_ratio=0.000004 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=19 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.021780 gemm_inner_pad_upper_bound=7112 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.237170521e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 8190, 0, 8192, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8190, 0, 8192, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 8190, 0, 0, 0, 0, 2, 8190] payload_rows_per_dst=[4, 8190, 8190, 8190, 0, 8190, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 8190, 8190, 8190, 0, 8190, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4443675016 prefetch_bytes=19730006016 reduce_bytes=10569646080 weight_movement_bytes=30299652096 egress_bytes=2222381448 ingress_bytes=32520945664 link_bytes=32520945664 total_nvlink_bytes=34743327112 weight_to_token_ratio=6.818602 prefetch_share_of_ingress=0.606686 dedup_ratio=0.002281 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=15 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023201 gemm_inner_pad_upper_bound=6477 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.138148255e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4343016880 prefetch_bytes=17616076800 reduce_bytes=16911433728 weight_movement_bytes=34527510528 egress_bytes=2172038576 ingress_bytes=36698488832 link_bytes=36698488832 total_nvlink_bytes=38870527408 weight_to_token_ratio=7.950121 prefetch_share_of_ingress=0.480022 dedup_ratio=0.000118 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=24 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.025095 gemm_inner_pad_upper_bound=6350 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.284351367e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 8192, 8192, 8192, 0, 8192, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 8192, 0, 8192, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 8192, 0, 8178, 8192, 8192, 14] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8192, 0, 8178, 8192, 8192, 14] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6487091504 prefetch_bytes=17616076800 reduce_bytes=19730006016 weight_movement_bytes=37346082816 egress_bytes=3244337456 ingress_bytes=40588836864 link_bytes=40588836864 total_nvlink_bytes=43833174320 weight_to_token_ratio=5.756984 prefetch_share_of_ingress=0.434013 dedup_ratio=0.000023 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=28 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.022727 gemm_inner_pad_upper_bound=6985 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.420503399e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6685746808 prefetch_bytes=17616076800 reduce_bytes=16911433728 weight_movement_bytes=34527510528 egress_bytes=3343689336 ingress_bytes=37869568000 link_bytes=37869568000 total_nvlink_bytes=41213257336 weight_to_token_ratio=5.164346 prefetch_share_of_ingress=0.465178 dedup_ratio=0.000004 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=24 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.025568 gemm_inner_pad_upper_bound=6223 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.325336083e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=20154 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1885, 6307, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 12614, 3770, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 12614, 3770, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7104394680 prefetch_bytes=16911433728 reduce_bytes=19025362944 weight_movement_bytes=35936796672 egress_bytes=3553064376 ingress_bytes=39488126976 link_bytes=39488126976 total_nvlink_bytes=43041191352 weight_to_token_ratio=5.058390 prefetch_share_of_ingress=0.428266 dedup_ratio=0.000011 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=27 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.024148 gemm_inner_pad_upper_bound=6731 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.381981424e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=165101568 dispatch_meta_bytes=80616 combine_payload_bytes=165101568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=165182184 ingress_bytes=1574387712 link_bytes=1574387712 token_bytes=330283752 weight_movement_bytes=1409286144 total_nvlink_bytes=1739569896 weight_to_token_ratio=4.266895 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.815914005e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16382, 8194, 0, 8192, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8194, 0, 8192, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201408504 ingress_bytes=4429168640 link_bytes=4429168640 token_bytes=402718712 weight_movement_bytes=4227858432 total_nvlink_bytes=4630577144 weight_to_token_ratio=10.498292 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480149556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6415444808 prefetch_bytes=14797504512 reduce_bytes=26776436736 weight_movement_bytes=41573941248 egress_bytes=3208506184 ingress_bytes=44780879872 link_bytes=44780879872 total_nvlink_bytes=47989386056 weight_to_token_ratio=6.480290 prefetch_share_of_ingress=0.330442 dedup_ratio=0.000797 prefetch_slots=42 unused_prefetch_slots=22 slot_utilization=0.656250 reduce_slots=38 remote_demand=21 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.026515 gemm_inner_pad_upper_bound=6477 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.567213968e-04 prefetch_bytes_per_gemm_flop=5.178740530e-05 anomalies=none - [2026-09-16 21:34:44] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 588.4 | throughput per GPU (TFLOP/s/GPU): 566.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.710403E+00 | loss scale: 1.0 | grad norm: 6.461 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=26900 dup_rows=66 dup_rows_remote=0 dedup_ratio=0.002014 router_tokens_per_expert=[1216, 1, 424, 2490, 6044, 6209, 0, 0] payload_rows_per_dst=[2432, 2, 848, 4980, 422, 5802, 6696, 11520] dup_rows_per_dst=[0, 0, 0, 0, 0, 66, 0, 0] combine_rows_per_src=[2432, 2, 848, 4980, 422, 5802, 6696, 11520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31876 payload_rows_remote=25594 dup_rows=892 dup_rows_remote=682 dedup_ratio=0.027222 router_tokens_per_expert=[962, 6, 1292, 2457, 5228, 6439, 0, 0] payload_rows_per_dst=[6282, 12882, 12366, 346, 0, 0, 0, 0] dup_rows_per_dst=[210, 8, 674, 0, 0, 0, 0, 0] combine_rows_per_src=[6282, 12882, 12366, 346, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=23258 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1219, 0, 448, 2435, 5655, 6627, 0, 0] payload_rows_per_dst=[2438, 0, 896, 4870, 9510, 13254, 0, 1800] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2438, 0, 896, 4870, 9510, 13254, 0, 1800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=220364800 dispatch_meta_bytes=107600 combine_payload_bytes=220364800 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=220472400 ingress_bytes=5152866304 link_bytes=5152866304 token_bytes=440837200 weight_movement_bytes=4932501504 total_nvlink_bytes=5373338704 weight_to_token_ratio=11.188941 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.885390039e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=209666048 dispatch_meta_bytes=105104 combine_payload_bytes=209666048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=209771152 ingress_bytes=1618952192 link_bytes=1618952192 token_bytes=419437200 weight_movement_bytes=1409286144 total_nvlink_bytes=1828723344 weight_to_token_ratio=3.359946 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.065456491e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=21572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1014, 4, 650, 2655, 5598, 6463, 0, 0] payload_rows_per_dst=[2028, 8, 1300, 4758, 552, 0, 12926, 11196] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2028, 8, 1300, 4758, 552, 0, 12926, 11196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32094 payload_rows_remote=28334 dup_rows=674 dup_rows_remote=674 dedup_ratio=0.020569 router_tokens_per_expert=[2259, 0, 1880, 2483, 4374, 5388, 0, 0] payload_rows_per_dst=[5918, 7410, 3760, 5076, 8748, 1182, 0, 0] dup_rows_per_dst=[408, 0, 0, 266, 0, 0, 0, 0] combine_rows_per_src=[5918, 7410, 3760, 5076, 8748, 1182, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=190529536 dispatch_meta_bytes=93032 combine_payload_bytes=190529536 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=190622568 ingress_bytes=4418387968 link_bytes=4418387968 token_bytes=381152104 weight_movement_bytes=4227858432 total_nvlink_bytes=4609010536 weight_to_token_ratio=11.092313 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.474112829e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32408 payload_rows_remote=19950 dup_rows=360 dup_rows_remote=360 dedup_ratio=0.010986 router_tokens_per_expert=[1710, 1, 605, 2627, 5210, 6228, 0, 3] payload_rows_per_dst=[6072, 12458, 8448, 2466, 2958, 0, 0, 6] dup_rows_per_dst=[136, 0, 224, 0, 0, 0, 0, 0] combine_rows_per_src=[6072, 12458, 8448, 2466, 2958, 0, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=28100 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000183 router_tokens_per_expert=[1549, 0, 980, 2244, 5289, 6322, 0, 0] payload_rows_per_dst=[3098, 0, 1960, 4662, 10578, 12464, 0, 0] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[3098, 0, 1960, 4662, 10578, 12464, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1873, 0, 1146, 2669, 4122, 6573, 0, 1] payload_rows_per_dst=[3746, 0, 2292, 5338, 0, 0, 13146, 8246] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3746, 0, 2292, 5338, 0, 0, 13146, 8246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=22846 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 6897, 8192, 0, 0, 1295, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 14024, 9922, 8822, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 14024, 9922, 8822, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7561, 8191, 0, 0, 630, 0, 2] payload_rows_per_dst=[16382, 15122, 0, 0, 0, 1260, 0, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 15122, 0, 0, 0, 1260, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=176717824 dispatch_meta_bytes=86288 combine_payload_bytes=176717824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=176804112 ingress_bytes=1586003968 link_bytes=1586003968 token_bytes=353521936 weight_movement_bytes=1409286144 total_nvlink_bytes=1762808080 weight_to_token_ratio=3.986418 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.880960190e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=232112128 dispatch_meta_bytes=116032 combine_payload_bytes=232112128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232228160 ingress_bytes=1641398272 link_bytes=1641398272 token_bytes=464340288 weight_movement_bytes=1409286144 total_nvlink_bytes=1873626432 weight_to_token_ratio=3.035029 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4096 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.191145170e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=163430400 dispatch_meta_bytes=81240 combine_payload_bytes=163430400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=163511640 ingress_bytes=1572716544 link_bytes=1572716544 token_bytes=326942040 weight_movement_bytes=1409286144 total_nvlink_bytes=1736228184 weight_to_token_ratio=4.310508 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.806556162e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=230195200 dispatch_meta_bytes=112400 combine_payload_bytes=230195200 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=230307600 ingress_bytes=3753410560 link_bytes=3753410560 token_bytes=460502800 weight_movement_bytes=3523215360 total_nvlink_bytes=3983718160 weight_to_token_ratio=7.650801 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.101753239e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=160743424 dispatch_meta_bytes=78488 combine_payload_bytes=160743424 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=160821912 ingress_bytes=1570029568 link_bytes=1570029568 token_bytes=321565336 weight_movement_bytes=1409286144 total_nvlink_bytes=1730851480 weight_to_token_ratio=4.382581 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.791510218e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7910, 8192, 0, 0, 282, 0, 0] payload_rows_per_dst=[0, 0, 6, 4172, 15068, 5332, 8190, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6, 4172, 15068, 5332, 8190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=187154432 dispatch_meta_bytes=91384 combine_payload_bytes=187154432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=187245816 ingress_bytes=1596440576 link_bytes=1596440576 token_bytes=374400248 weight_movement_bytes=1409286144 total_nvlink_bytes=1783686392 weight_to_token_ratio=3.764116 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2560 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.939400839e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=144998400 dispatch_meta_bytes=70800 combine_payload_bytes=144998400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=145069200 ingress_bytes=1554284544 link_bytes=1554284544 token_bytes=290067600 weight_movement_bytes=1409286144 total_nvlink_bytes=1699353744 weight_to_token_ratio=4.858475 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.703344656e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16382 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 7504, 8191, 0, 1, 686, 0, 2] payload_rows_per_dst=[0, 0, 0, 0, 2, 8248, 8132, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[0, 0, 0, 0, 2, 8248, 8132, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 6967, 0, 1225, 8192, 0, 0] payload_rows_per_dst=[0, 0, 862, 1310, 6260, 16384, 2722, 5230] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 862, 1310, 6260, 16384, 2722, 5230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 6584, 0, 1608, 8192, 0, 0] payload_rows_per_dst=[16384, 7650, 6634, 0, 2100, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 7650, 6634, 0, 2100, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18462 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7153, 8190, 0, 1, 1039, 1, 0] payload_rows_per_dst=[0, 0, 16380, 14306, 2, 2078, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16380, 14306, 2, 2078, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16386 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 8125, 8191, 0, 0, 68, 0, 0] payload_rows_per_dst=[4, 1956, 16378, 14290, 0, 136, 0, 0] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 1956, 16378, 14290, 0, 136, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25146 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7902, 8191, 0, 0, 290, 0, 1] payload_rows_per_dst=[0, 0, 0, 0, 3672, 5096, 7622, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 3672, 5096, 7622, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30278 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 6947, 0, 1245, 8192, 0, 0] payload_rows_per_dst=[0, 0, 6894, 7000, 2490, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6894, 7000, 2490, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134201344 dispatch_meta_bytes=65528 combine_payload_bytes=134201344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134266872 ingress_bytes=1543487488 link_bytes=1543487488 token_bytes=268468216 weight_movement_bytes=1409286144 total_nvlink_bytes=1677754360 weight_to_token_ratio=5.249359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642885649e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=151240704 dispatch_meta_bytes=73848 combine_payload_bytes=151240704 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=151314552 ingress_bytes=1560526848 link_bytes=1560526848 token_bytes=302555256 weight_movement_bytes=1409286144 total_nvlink_bytes=1711841400 weight_to_token_ratio=4.657946 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.738298953e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=5066735616 link_bytes=5066735616 token_bytes=268533768 weight_movement_bytes=4932501504 total_nvlink_bytes=5201035272 weight_to_token_ratio=18.368273 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837160449e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=205996032 dispatch_meta_bytes=100584 combine_payload_bytes=205996032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=206096616 ingress_bytes=1615282176 link_bytes=1615282176 token_bytes=412092648 weight_movement_bytes=1409286144 total_nvlink_bytes=1821378792 weight_to_token_ratio=3.419828 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.044905934e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=248037376 dispatch_meta_bytes=121112 combine_payload_bytes=248037376 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=248158488 ingress_bytes=1657323520 link_bytes=1657323520 token_bytes=496195864 weight_movement_bytes=1409286144 total_nvlink_bytes=1905482008 weight_to_token_ratio=2.840181 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.280319911e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17078 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7845, 8191, 0, 0, 348, 0, 0] payload_rows_per_dst=[16382, 15690, 0, 0, 0, 696, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 15690, 0, 0, 0, 696, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8190, 0, 7567, 627, 0] payload_rows_per_dst=[16380, 15134, 0, 0, 0, 0, 1254, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 15134, 0, 0, 0, 0, 1254, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 1, 6896, 1295, 0] payload_rows_per_dst=[0, 0, 0, 16384, 2, 0, 14444, 1938] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 2, 0, 14444, 1938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 1, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16974 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 6885, 0, 1307, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 2614, 0, 14360, 15794] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 2614, 0, 14360, 15794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19246 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 6414, 0, 1778, 8192, 0, 0] payload_rows_per_dst=[0, 8192, 7498, 13522, 3556, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 7498, 13522, 3556, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 4823, 0, 3369, 8192, 0, 0] payload_rows_per_dst=[0, 9972, 5122, 10936, 6738, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 9972, 5122, 10936, 6738, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17082 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 5523, 0, 2669, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 5338, 0, 15686, 11744] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 5338, 0, 15686, 11744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=139902976 dispatch_meta_bytes=68312 combine_payload_bytes=139902976 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=139971288 ingress_bytes=5072404480 link_bytes=5072404480 token_bytes=279874264 weight_movement_bytes=4932501504 total_nvlink_bytes=5212375768 weight_to_token_ratio=17.623991 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.840334776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 7913, 279, 0] payload_rows_per_dst=[0, 0, 18, 16366, 0, 4180, 12204, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 18, 16366, 0, 4180, 12204, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=268566528 ingress_bytes=5905580032 link_bytes=5905580032 token_bytes=537001984 weight_movement_bytes=5637144576 total_nvlink_bytes=6174146560 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306878307e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=139051008 dispatch_meta_bytes=67896 combine_payload_bytes=139051008 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=139118904 ingress_bytes=1548337152 link_bytes=1548337152 token_bytes=278169912 weight_movement_bytes=1409286144 total_nvlink_bytes=1687456056 weight_to_token_ratio=5.066278 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.670041743e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=157663232 dispatch_meta_bytes=76984 combine_payload_bytes=157663232 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=157740216 ingress_bytes=1566949376 link_bytes=1566949376 token_bytes=315403448 weight_movement_bytes=1409286144 total_nvlink_bytes=1724689592 weight_to_token_ratio=4.468201 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.774262429e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=226476032 dispatch_meta_bytes=110584 combine_payload_bytes=226476032 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=226586616 ingress_bytes=4454334464 link_bytes=4454334464 token_bytes=453062648 weight_movement_bytes=4227858432 total_nvlink_bytes=4680921080 weight_to_token_ratio=9.331730 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.494241367e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=139935744 dispatch_meta_bytes=68328 combine_payload_bytes=139935744 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=140004072 ingress_bytes=1549221888 link_bytes=1549221888 token_bytes=279939816 weight_movement_bytes=1409286144 total_nvlink_bytes=1689225960 weight_to_token_ratio=5.034247 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.674995896e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 8192, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 6356, 0, 1836, 8192, 0, 0] payload_rows_per_dst=[16384, 6954, 5758, 0, 3672, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 6954, 5758, 0, 3672, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8188, 0, 7153, 1043, 0] payload_rows_per_dst=[0, 0, 16376, 0, 0, 14306, 2086, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16376, 0, 0, 14306, 2086, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17742 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8190, 0, 7513, 681, 0] payload_rows_per_dst=[0, 0, 0, 0, 16380, 0, 1362, 15026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16380, 0, 1362, 15026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32186 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8191, 0, 7902, 291, 0] payload_rows_per_dst=[0, 0, 0, 4, 16378, 0, 582, 15804] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 4, 16378, 0, 582, 15804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 2, 0, 16384, 16382, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 16384, 16382, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16394 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 4, 8119, 69, 0] payload_rows_per_dst=[10, 1942, 16374, 14, 8, 14282, 138, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10, 1942, 16374, 14, 8, 14282, 138, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=211468288 dispatch_meta_bytes=103256 combine_payload_bytes=211468288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=211571544 ingress_bytes=1620754432 link_bytes=1620754432 token_bytes=423039832 weight_movement_bytes=1409286144 total_nvlink_bytes=1832325976 weight_to_token_ratio=3.331332 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.075548283e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=5905580032 link_bytes=5905580032 token_bytes=537001984 weight_movement_bytes=5637144576 total_nvlink_bytes=6174146560 weight_to_token_ratio=10.497437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306878307e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=145342464 dispatch_meta_bytes=70968 combine_payload_bytes=145342464 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=145413432 ingress_bytes=1554628608 link_bytes=1554628608 token_bytes=290755896 weight_movement_bytes=1409286144 total_nvlink_bytes=1700042040 weight_to_token_ratio=4.846974 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.705271271e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=263667712 dispatch_meta_bytes=128744 combine_payload_bytes=263667712 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=263796456 ingress_bytes=1672953856 link_bytes=1672953856 token_bytes=527464168 weight_movement_bytes=1409286144 total_nvlink_bytes=1936750312 weight_to_token_ratio=2.671814 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2560 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.367843268e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134299648 dispatch_meta_bytes=65576 combine_payload_bytes=134299648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134365224 ingress_bytes=1543585792 link_bytes=1543585792 token_bytes=268664872 weight_movement_bytes=1409286144 total_nvlink_bytes=1677951016 weight_to_token_ratio=5.245517 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643436111e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 26, 8192, 8166, 0, 0] payload_rows_per_dst=[16384, 16332, 0, 52, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16332, 0, 52, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16404 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16370, 16364, 2, 32] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16370, 16364, 2, 32] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17076 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8189, 0, 7846, 349, 0] payload_rows_per_dst=[16378, 15692, 0, 0, 0, 0, 698, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 15692, 0, 0, 0, 0, 698, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8191, 0, 8192, 0, 1, 0, 0] payload_rows_per_dst=[0, 8192, 8190, 16384, 0, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8190, 16384, 0, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134381568 dispatch_meta_bytes=65616 combine_payload_bytes=134381568 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134447184 ingress_bytes=5771526144 link_bytes=5771526144 token_bytes=268828752 weight_movement_bytes=5637144576 total_nvlink_bytes=5905973328 weight_to_token_ratio=20.969277 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231813725e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 5, 8192, 8187, 0, 0] payload_rows_per_dst=[0, 0, 0, 10, 16384, 16374, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 10, 16384, 16374, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=139886592 dispatch_meta_bytes=68304 combine_payload_bytes=139886592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=139954896 ingress_bytes=1549172736 link_bytes=1549172736 token_bytes=279841488 weight_movement_bytes=1409286144 total_nvlink_bytes=1689127632 weight_to_token_ratio=5.036016 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.674720665e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[0, 8192, 8192, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134283264 ingress_bytes=5771362304 link_bytes=5771362304 token_bytes=268500992 weight_movement_bytes=5637144576 total_nvlink_bytes=5905645568 weight_to_token_ratio=20.994874 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231721982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 16382, 0, 16384, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16382, 0, 16384, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 8192, 8192, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 8192, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16414 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 4, 9, 8189, 8182, 0, 0] payload_rows_per_dst=[0, 0, 16370, 16352, 14, 30, 0, 0] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16370, 16352, 14, 30, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 6, 8191, 8185, 0, 0] payload_rows_per_dst=[0, 0, 4, 12, 0, 0, 16382, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 12, 0, 0, 16382, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16402 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3, 6, 8192, 8183, 0, 0] payload_rows_per_dst=[0, 0, 6, 12, 0, 0, 16384, 16366] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 6, 12, 0, 0, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[2, 0, 0, 16382, 0, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 16382, 0, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=4429185024 link_bytes=4429185024 token_bytes=402751488 weight_movement_bytes=4227858432 total_nvlink_bytes=4630609920 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480158730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=16386 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 3, 6, 8190, 8185, 0, 0] payload_rows_per_dst=[4, 80, 16380, 16302, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 80, 16380, 16302, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134463488 dispatch_meta_bytes=65664 combine_payload_bytes=134463488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134529152 ingress_bytes=1543749632 link_bytes=1543749632 token_bytes=268992640 weight_movement_bytes=1409286144 total_nvlink_bytes=1678278784 weight_to_token_ratio=5.239125 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644353546e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 2047, 0, 6145, 8192, 0] payload_rows_per_dst=[16384, 5280, 7010, 4094, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 5280, 7010, 4094, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134365184 dispatch_meta_bytes=65608 combine_payload_bytes=134365184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134430792 ingress_bytes=1543651328 link_bytes=1543651328 token_bytes=268795976 weight_movement_bytes=1409286144 total_nvlink_bytes=1678082120 weight_to_token_ratio=5.242958 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643803085e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29302 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 2840, 0, 5352, 8192, 0] payload_rows_per_dst=[0, 0, 0, 5680, 11400, 3466, 8192, 4030] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 5680, 11400, 3466, 8192, 4030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=240041984 dispatch_meta_bytes=117208 combine_payload_bytes=240041984 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=240159192 ingress_bytes=4467900416 link_bytes=4467900416 token_bytes=480201176 weight_movement_bytes=4227858432 total_nvlink_bytes=4708059608 weight_to_token_ratio=8.804348 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.501837734e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16412 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 14, 8190, 8178, 0, 0] payload_rows_per_dst=[16380, 16356, 4, 28, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 16356, 4, 28, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[2, 16378, 0, 4, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 16378, 0, 4, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[2, 0, 0, 0, 0, 0, 16382, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 0, 0, 0, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18194 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 1965, 0, 6227, 8192, 0] payload_rows_per_dst=[0, 0, 0, 3930, 14574, 5158, 8192, 914] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 3930, 14574, 5158, 8192, 914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134447104 dispatch_meta_bytes=65648 combine_payload_bytes=134447104 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134512752 ingress_bytes=1543733248 link_bytes=1543733248 token_bytes=268959856 weight_movement_bytes=1409286144 total_nvlink_bytes=1678246000 weight_to_token_ratio=5.239764 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644261803e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 16384, 0, 0, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 16384, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=149045248 dispatch_meta_bytes=72776 combine_payload_bytes=149045248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=149118024 ingress_bytes=1558331392 link_bytes=1558331392 token_bytes=298163272 weight_movement_bytes=1409286144 total_nvlink_bytes=1707449416 weight_to_token_ratio=4.726558 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.726005316e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 8189, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16378, 6, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 6, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 2180, 0, 6012, 8192, 0] payload_rows_per_dst=[0, 0, 0, 4360, 0, 6344, 8192, 13872] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 4360, 0, 6344, 8192, 13872] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27738 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 2515, 0, 5677, 8192, 0] payload_rows_per_dst=[0, 8192, 8192, 5030, 5190, 6164, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8192, 8192, 5030, 5190, 6164, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18816 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 2025, 0, 6167, 8192, 0] payload_rows_per_dst=[0, 0, 0, 2752, 0, 7872, 8192, 13952] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 2752, 0, 7872, 8192, 13952] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 16384, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19320 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3505, 0, 0, 0, 4687, 8192, 0, 0] payload_rows_per_dst=[13448, 16384, 0, 2936, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13448, 16384, 0, 2936, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268386304 dispatch_meta_bytes=131048 combine_payload_bytes=268386304 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268517352 ingress_bytes=4496244736 link_bytes=4496244736 token_bytes=536903656 weight_movement_bytes=4227858432 total_nvlink_bytes=4764762088 weight_to_token_ratio=7.874520 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517709370e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=21342 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 1173, 0, 7019, 8192, 0] payload_rows_per_dst=[0, 12828, 11426, 3146, 1604, 3764, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 12828, 11426, 3146, 1604, 3764, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=4429185024 link_bytes=4429185024 token_bytes=402751488 weight_movement_bytes=4227858432 total_nvlink_bytes=4630609920 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480158730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=227229696 dispatch_meta_bytes=110952 combine_payload_bytes=227229696 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=227340648 ingress_bytes=1636515840 link_bytes=1636515840 token_bytes=454570344 weight_movement_bytes=1409286144 total_nvlink_bytes=1863856488 weight_to_token_ratio=3.100260 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.163805589e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3002, 0, 0, 0, 5190, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 10380, 16384, 0, 6004] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 10380, 16384, 0, 6004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=154140672 dispatch_meta_bytes=75264 combine_payload_bytes=154140672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=154215936 ingress_bytes=1563426816 link_bytes=1563426816 token_bytes=308356608 weight_movement_bytes=1409286144 total_nvlink_bytes=1717642752 weight_to_token_ratio=4.570313 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.754537563e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=158269440 dispatch_meta_bytes=77280 combine_payload_bytes=158269440 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=158346720 ingress_bytes=2976841728 link_bytes=2976841728 token_bytes=316616160 weight_movement_bytes=2818572288 total_nvlink_bytes=3135188448 weight_to_token_ratio=8.902174 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7680 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.666907108e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=174833664 dispatch_meta_bytes=85368 combine_payload_bytes=174833664 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=174919032 ingress_bytes=1584119808 link_bytes=1584119808 token_bytes=349752696 weight_movement_bytes=1409286144 total_nvlink_bytes=1759038840 weight_to_token_ratio=4.029379 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.870409681e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26300 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 1888, 0, 6304, 8192, 0] payload_rows_per_dst=[16384, 6468, 6140, 3776, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 6468, 6140, 3776, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 16384, 0, 0, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27738 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2961, 0, 0, 0, 5231, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 5432, 5030, 16384, 0, 5922] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 5432, 5030, 16384, 0, 5922] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=215449600 dispatch_meta_bytes=105200 combine_payload_bytes=215449600 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=215554800 ingress_bytes=1624735744 link_bytes=1624735744 token_bytes=431004400 weight_movement_bytes=1409286144 total_nvlink_bytes=1840290544 weight_to_token_ratio=3.269772 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.097841968e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[0, 16384, 0, 0, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 4935, 108, 3149] payload_rows_per_dst=[16384, 0, 0, 0, 9870, 2484, 4030, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 0, 0, 9870, 2484, 4030, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25894 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 5662, 128, 2402] payload_rows_per_dst=[0, 0, 16360, 24, 0, 6874, 4706, 4804] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16360, 24, 0, 6874, 4706, 4804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=227229696 dispatch_meta_bytes=110952 combine_payload_bytes=227229696 prefetch_bytes=0 reduce_bytes=2818572288 egress_bytes=227340648 ingress_bytes=3045801984 link_bytes=3045801984 token_bytes=454570344 weight_movement_bytes=2818572288 total_nvlink_bytes=3273142632 weight_to_token_ratio=6.200520 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.705521973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=212123648 dispatch_meta_bytes=103576 combine_payload_bytes=212123648 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=212227224 ingress_bytes=4439982080 link_bytes=4439982080 token_bytes=424350872 weight_movement_bytes=4227858432 total_nvlink_bytes=4652209304 weight_to_token_ratio=9.963119 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.486204631e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 0, 16384, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16384, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4355, 0, 0, 0, 3837, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 7674, 0, 16384, 8710] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 7674, 0, 16384, 8710] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=23190 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3403, 0, 0, 0, 4789, 8192, 0, 0] payload_rows_per_dst=[2240, 0, 16384, 9578, 0, 0, 0, 4566] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2240, 0, 16384, 9578, 0, 0, 0, 4566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26068 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3350, 0, 0, 0, 4842, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 9684, 0, 16384, 6700] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 9684, 0, 16384, 6700] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 4798, 161, 3233] payload_rows_per_dst=[0, 0, 16384, 0, 0, 9596, 976, 5812] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 0, 0, 9596, 976, 5812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5074, 0, 0, 0, 3118, 8192, 0, 0] payload_rows_per_dst=[9282, 0, 16384, 6236, 0, 0, 0, 866] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[9282, 0, 16384, 6236, 0, 0, 0, 866] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=189972480 dispatch_meta_bytes=92760 combine_payload_bytes=189972480 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=190065240 ingress_bytes=1599258624 link_bytes=1599258624 token_bytes=380037720 weight_movement_bytes=1409286144 total_nvlink_bytes=1789323864 weight_to_token_ratio=3.708280 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.955180731e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=213549056 dispatch_meta_bytes=104272 combine_payload_bytes=213549056 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=213653328 ingress_bytes=1622835200 link_bytes=1622835200 token_bytes=427202384 weight_movement_bytes=1409286144 total_nvlink_bytes=1836488528 weight_to_token_ratio=3.298872 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.087199715e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 2, 0, 0, 5502, 99, 2589] payload_rows_per_dst=[0, 0, 4, 16384, 0, 3892, 6376, 6112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 16384, 0, 3892, 6376, 6112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3899, 0, 0, 0, 4293, 8192, 0, 0] payload_rows_per_dst=[7798, 16384, 0, 8586, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7798, 16384, 0, 8586, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 2, 0, 0, 5100, 232, 2858] payload_rows_per_dst=[0, 16384, 4, 0, 5214, 4986, 4546, 1634] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 4, 0, 5214, 4986, 4546, 1634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=21478 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8192, 4, 0, 0, 5352, 154, 2682] payload_rows_per_dst=[0, 0, 8, 16360, 0, 10, 5100, 11288] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[0, 0, 8, 16360, 0, 10, 5100, 11288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=216203264 dispatch_meta_bytes=105568 combine_payload_bytes=216203264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=216308832 ingress_bytes=1625489408 link_bytes=1625489408 token_bytes=432512096 weight_movement_bytes=1409286144 total_nvlink_bytes=1841798240 weight_to_token_ratio=3.258374 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.102062172e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 3680, 88, 4424] payload_rows_per_dst=[0, 16384, 0, 0, 7360, 2246, 3662, 3116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 7360, 2246, 3662, 3116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=21330 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 896, 7296, 0, 0] payload_rows_per_dst=[0, 0, 0, 16384, 1792, 11438, 3154, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 1792, 11438, 3154, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 1156, 7036, 0, 0] payload_rows_per_dst=[16384, 14072, 0, 0, 2312, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 14072, 0, 0, 2312, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=175947776 dispatch_meta_bytes=85912 combine_payload_bytes=175947776 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=176033688 ingress_bytes=2994520064 link_bytes=2994520064 token_bytes=351981464 weight_movement_bytes=2818572288 total_nvlink_bytes=3170553752 weight_to_token_ratio=8.007729 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.676806238e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=174735360 dispatch_meta_bytes=85320 combine_payload_bytes=174735360 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=174820680 ingress_bytes=4402593792 link_bytes=4402593792 token_bytes=349556040 weight_movement_bytes=4227858432 total_nvlink_bytes=4577414472 weight_to_token_ratio=12.094937 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.465268750e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 4, 0, 0, 5162, 125, 2901] payload_rows_per_dst=[16384, 0, 8, 0, 10324, 2680, 3372, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 8, 0, 10324, 2680, 3372, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 16384, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30596 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 1086, 7106, 0, 0] payload_rows_per_dst=[0, 0, 0, 16384, 2172, 14212, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 2172, 14212, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=268566528 ingress_bytes=5905580032 link_bytes=5905580032 token_bytes=537001984 weight_movement_bytes=5637144576 total_nvlink_bytes=6174146560 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306878307e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16396 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8186, 0, 6, 0] payload_rows_per_dst=[0, 0, 0, 16384, 0, 16372, 12, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 16372, 12, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 16384, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8185, 0, 7, 0] payload_rows_per_dst=[16384, 0, 16370, 0, 0, 0, 14, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16370, 0, 0, 0, 14, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=250642432 dispatch_meta_bytes=122384 combine_payload_bytes=250642432 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=250764816 ingress_bytes=1659928576 link_bytes=1659928576 token_bytes=501407248 weight_movement_bytes=1409286144 total_nvlink_bytes=1910693392 weight_to_token_ratio=2.810662 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3968 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.294907137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134316032 dispatch_meta_bytes=65584 combine_payload_bytes=134316032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134381616 ingress_bytes=1543602176 link_bytes=1543602176 token_bytes=268697648 weight_movement_bytes=1409286144 total_nvlink_bytes=1677983792 weight_to_token_ratio=5.244877 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643527854e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18056 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 836, 7356, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 1672, 0, 14712, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 1672, 0, 14712, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 1084, 7108, 0, 0] payload_rows_per_dst=[0, 0, 16384, 0, 9266, 7118, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 0, 9266, 7118, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=147914752 dispatch_meta_bytes=72224 combine_payload_bytes=147914752 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=147986976 ingress_bytes=1557200896 link_bytes=1557200896 token_bytes=295901728 weight_movement_bytes=1409286144 total_nvlink_bytes=1705187872 weight_to_token_ratio=4.762683 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.719675010e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32666 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8179, 0, 13, 0] payload_rows_per_dst=[0, 0, 0, 16384, 102, 16256, 26, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 102, 16256, 26, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 0, 0, 16384, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 803, 7389, 0, 0] payload_rows_per_dst=[0, 3986, 16384, 0, 12398, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 3986, 16384, 0, 12398, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 741, 7451, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 1482, 0, 14902, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 1482, 0, 14902, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=267599872 dispatch_meta_bytes=130664 combine_payload_bytes=267599872 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=267730536 ingress_bytes=4495458304 link_bytes=4495458304 token_bytes=535330408 weight_movement_bytes=4227858432 total_nvlink_bytes=4763188840 weight_to_token_ratio=7.897662 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517269001e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16524 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8182, 0, 10, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 140, 16244, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 140, 16244, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18058 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 0, 837, 7355, 0, 0] payload_rows_per_dst=[16384, 14710, 0, 0, 1674, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 14710, 0, 0, 1674, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8189, 0, 3, 0] payload_rows_per_dst=[0, 16384, 0, 0, 16378, 0, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 16378, 0, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=135364608 dispatch_meta_bytes=66096 combine_payload_bytes=135364608 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135430704 ingress_bytes=1544650752 link_bytes=1544650752 token_bytes=270795312 weight_movement_bytes=1409286144 total_nvlink_bytes=1680081456 weight_to_token_ratio=5.204249 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.649399442e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 0, 0, 16384, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17158 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 7805, 0, 0, 387] payload_rows_per_dst=[0, 0, 0, 16384, 0, 15610, 0, 774] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 15610, 0, 774] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=147931136 dispatch_meta_bytes=72232 combine_payload_bytes=147931136 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=148003368 ingress_bytes=1557217280 link_bytes=1557217280 token_bytes=295934504 weight_movement_bytes=1409286144 total_nvlink_bytes=1705220648 weight_to_token_ratio=4.762156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.719766754e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32720 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8168, 0, 24, 0] payload_rows_per_dst=[0, 16384, 48, 0, 16288, 0, 48, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 48, 0, 16288, 0, 48, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8177, 0, 15, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 7879, 0, 0, 313] payload_rows_per_dst=[16384, 0, 15758, 0, 0, 0, 0, 626] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 15758, 0, 0, 0, 0, 626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=140558336 dispatch_meta_bytes=68632 combine_payload_bytes=140558336 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=140626968 ingress_bytes=1549844480 link_bytes=1549844480 token_bytes=281185304 weight_movement_bytes=1409286144 total_nvlink_bytes=1690471448 weight_to_token_ratio=5.011948 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.678482151e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268042240 dispatch_meta_bytes=130880 combine_payload_bytes=268042240 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268173120 ingress_bytes=1677328384 link_bytes=1677328384 token_bytes=536215360 weight_movement_bytes=1409286144 total_nvlink_bytes=1945501504 weight_to_token_ratio=2.628209 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.392338799e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=5841568152 prefetch_bytes=14092861440 reduce_bytes=37346082816 weight_movement_bytes=51438944256 egress_bytes=2921496984 ingress_bytes=54359015424 link_bytes=54359015424 total_nvlink_bytes=57280512408 weight_to_token_ratio=8.805674 prefetch_share_of_ingress=0.259255 dedup_ratio=0.000126 prefetch_slots=40 unused_prefetch_slots=24 slot_utilization=0.625000 reduce_slots=53 remote_demand=20 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.026042 gemm_inner_pad_upper_bound=6731 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.902423724e-04 prefetch_bytes_per_gemm_flop=4.932133838e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 16384, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8175, 0, 17, 0] payload_rows_per_dst=[16384, 0, 16350, 0, 0, 0, 34, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16350, 0, 0, 0, 34, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4495165576 prefetch_bytes=21139292160 reduce_bytes=5637144576 weight_movement_bytes=26776436736 egress_bytes=2248132744 ingress_bytes=29023469568 link_bytes=29023469568 total_nvlink_bytes=31271602312 weight_to_token_ratio=5.956719 prefetch_share_of_ingress=0.728352 dedup_ratio=0.001701 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=8 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.027462 gemm_inner_pad_upper_bound=5080 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.015745716e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30220 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 7761, 0, 0, 431] payload_rows_per_dst=[0, 0, 0, 16384, 2548, 12974, 0, 862] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 2548, 12974, 0, 862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 16384, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=247562240 dispatch_meta_bytes=120880 combine_payload_bytes=247562240 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=247683120 ingress_bytes=4475420672 link_bytes=4475420672 token_bytes=495245360 weight_movement_bytes=4227858432 total_nvlink_bytes=4723103792 weight_to_token_ratio=8.536897 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.506048763e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 8020, 0, 0, 172] payload_rows_per_dst=[0, 0, 0, 0, 0, 4184, 16384, 12200] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 4184, 16384, 12200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6674144104 prefetch_bytes=14092861440 reduce_bytes=25367150592 weight_movement_bytes=39460012032 egress_bytes=3337886568 ingress_bytes=42796269568 link_bytes=42796269568 total_nvlink_bytes=46134156136 weight_to_token_ratio=5.912370 prefetch_share_of_ingress=0.329301 dedup_ratio=0.000000 prefetch_slots=40 unused_prefetch_slots=24 slot_utilization=0.625000 reduce_slots=36 remote_demand=20 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.026989 gemm_inner_pad_upper_bound=6477 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.497757785e-04 prefetch_bytes_per_gemm_flop=4.932133838e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 7839, 0, 0, 353] payload_rows_per_dst=[0, 16384, 0, 0, 15678, 0, 0, 706] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 0, 15678, 0, 0, 706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 0, 0, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=31638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 7836, 0, 0, 356] payload_rows_per_dst=[0, 16384, 1130, 0, 14542, 0, 0, 712] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 1130, 0, 14542, 0, 0, 712] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=5628065288 prefetch_bytes=18320719872 reduce_bytes=12683575296 weight_movement_bytes=31004295168 egress_bytes=2814719496 ingress_bytes=33817640960 link_bytes=33817640960 total_nvlink_bytes=36632360456 weight_to_token_ratio=5.508873 prefetch_share_of_ingress=0.541750 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=18 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.026989 gemm_inner_pad_upper_bound=5715 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.183529207e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 7804, 0, 0, 388] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6746513520 prefetch_bytes=20434649088 reduce_bytes=11274289152 weight_movement_bytes=31708938240 egress_bytes=3374080112 ingress_bytes=35081371648 link_bytes=35081371648 total_nvlink_bytes=38455451760 weight_to_token_ratio=4.700048 prefetch_share_of_ingress=0.582493 dedup_ratio=0.000015 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=16 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.026989 gemm_inner_pad_upper_bound=5334 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.227756485e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=259178496 dispatch_meta_bytes=126552 combine_payload_bytes=259178496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=259305048 ingress_bytes=1668464640 link_bytes=1668464640 token_bytes=518483544 weight_movement_bytes=1409286144 total_nvlink_bytes=1927769688 weight_to_token_ratio=2.718092 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.342705533e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6215348352 prefetch_bytes=19730006016 reduce_bytes=11978932224 weight_movement_bytes=31708938240 egress_bytes=3108434048 ingress_bytes=34815852544 link_bytes=34815852544 total_nvlink_bytes=37924286592 weight_to_token_ratio=5.101715 prefetch_share_of_ingress=0.566696 dedup_ratio=0.001297 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=17 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.026515 gemm_inner_pad_upper_bound=5588 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.218464009e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4695260328 prefetch_bytes=19730006016 reduce_bytes=9865003008 weight_movement_bytes=29595009024 egress_bytes=2348203176 ingress_bytes=31942066176 link_bytes=31942066176 total_nvlink_bytes=34290269352 weight_to_token_ratio=6.303167 prefetch_share_of_ingress=0.617681 dedup_ratio=0.000011 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=14 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.026515 gemm_inner_pad_upper_bound=5588 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.117888983e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 0, 0, 7940, 0, 0, 252] payload_rows_per_dst=[16384, 0, 15880, 0, 0, 0, 0, 504] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 15880, 0, 0, 0, 0, 504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6198762440 prefetch_bytes=14092861440 reduce_bytes=27481079808 weight_movement_bytes=41573941248 egress_bytes=3100138440 ingress_bytes=44672565248 link_bytes=44672565248 total_nvlink_bytes=47772703688 weight_to_token_ratio=6.706813 prefetch_share_of_ingress=0.315470 dedup_ratio=0.000687 prefetch_slots=40 unused_prefetch_slots=24 slot_utilization=0.625000 reduce_slots=39 remote_demand=20 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.029356 gemm_inner_pad_upper_bound=5842 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.563423238e-04 prefetch_bytes_per_gemm_flop=4.932133838e-05 anomalies=none - [2026-09-16 21:34:44] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 585.4 | throughput per GPU (TFLOP/s/GPU): 569.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.490894E+00 | loss scale: 1.0 | grad norm: 5.786 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=26808 dup_rows=84 dup_rows_remote=84 dedup_ratio=0.002563 router_tokens_per_expert=[1317, 2, 2298, 5377, 1437, 5953, 0, 0] payload_rows_per_dst=[2634, 4, 2762, 58, 4624, 5876, 6030, 10696] dup_rows_per_dst=[0, 0, 0, 0, 84, 0, 0, 0] combine_rows_per_src=[2634, 4, 2762, 58, 4624, 5876, 6030, 10696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30764 payload_rows_remote=24066 dup_rows=2004 dup_rows_remote=1490 dedup_ratio=0.061157 router_tokens_per_expert=[1322, 9, 2737, 5161, 988, 6167, 0, 0] payload_rows_per_dst=[3140, 7014, 6698, 9612, 3830, 470, 0, 0] dup_rows_per_dst=[156, 6, 514, 1116, 212, 0, 0, 0] combine_rows_per_src=[3140, 7014, 6698, 9612, 3830, 470, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28014 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[241, 1, 1499, 5214, 2377, 7052, 0, 0] payload_rows_per_dst=[482, 2, 2998, 10318, 4754, 14104, 0, 110] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[482, 2, 2998, 10318, 4754, 14104, 0, 110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32510 payload_rows_remote=20918 dup_rows=258 dup_rows_remote=2 dedup_ratio=0.007874 router_tokens_per_expert=[572, 7, 2459, 5352, 1431, 6562, 0, 1] payload_rows_per_dst=[11592, 13136, 4918, 0, 2862, 0, 0, 2] dup_rows_per_dst=[256, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[11592, 13136, 4918, 0, 2862, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 2, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32724 payload_rows_remote=21706 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.001343 router_tokens_per_expert=[1115, 5, 2372, 5509, 711, 6672, 0, 0] payload_rows_per_dst=[2230, 10, 2412, 0, 3710, 0, 13344, 11018] dup_rows_per_dst=[0, 0, 0, 0, 44, 0, 0, 0] combine_rows_per_src=[2230, 10, 2412, 0, 3710, 0, 13344, 11018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=219611136 dispatch_meta_bytes=107568 combine_payload_bytes=219611136 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=219718704 ingress_bytes=5856755712 link_bytes=5856755712 token_bytes=439329840 weight_movement_bytes=5637144576 total_nvlink_bytes=6076474416 weight_to_token_ratio=12.831235 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.279538726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32738 payload_rows_remote=19344 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000916 router_tokens_per_expert=[1498, 1, 1372, 5469, 1347, 6697, 0, 0] payload_rows_per_dst=[2996, 2, 1164, 0, 4244, 0, 13394, 10938] dup_rows_per_dst=[0, 0, 0, 0, 30, 0, 0, 0] combine_rows_per_src=[2996, 2, 1164, 0, 4244, 0, 13394, 10938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=197148672 dispatch_meta_bytes=102224 combine_payload_bytes=197148672 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=197250896 ingress_bytes=2311077888 link_bytes=2311077888 token_bytes=394399568 weight_movement_bytes=2113929216 total_nvlink_bytes=2508328784 weight_to_token_ratio=5.359867 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=7808 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.294107148e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32634 payload_rows_remote=20970 dup_rows=134 dup_rows_remote=134 dedup_ratio=0.004089 router_tokens_per_expert=[499, 1, 1885, 5832, 1403, 6764, 0, 0] payload_rows_per_dst=[998, 2, 3770, 11664, 3882, 12318, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 134, 0, 0, 0] combine_rows_per_src=[998, 2, 3770, 11664, 3882, 12318, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=229490688 dispatch_meta_bytes=112056 combine_payload_bytes=229490688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=229602744 ingress_bytes=1638776832 link_bytes=1638776832 token_bytes=459093432 weight_movement_bytes=1409286144 total_nvlink_bytes=1868379576 weight_to_token_ratio=3.069715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6656 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.176466200e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=171360256 dispatch_meta_bytes=83680 combine_payload_bytes=171360256 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=171443936 ingress_bytes=1580646400 link_bytes=1580646400 token_bytes=342804192 weight_movement_bytes=1409286144 total_nvlink_bytes=1752090336 weight_to_token_ratio=4.111053 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3968 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.850960046e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=177815552 dispatch_meta_bytes=87000 combine_payload_bytes=177815552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=177902552 ingress_bytes=1587101696 link_bytes=1587101696 token_bytes=355718104 weight_movement_bytes=1409286144 total_nvlink_bytes=1765004248 weight_to_token_ratio=3.961806 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.887107009e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32090 payload_rows_remote=19524 dup_rows=678 dup_rows_remote=654 dedup_ratio=0.020691 router_tokens_per_expert=[625, 19, 2570, 4713, 2179, 6276, 0, 2] payload_rows_per_dst=[8090, 12566, 7072, 0, 4358, 0, 0, 4] dup_rows_per_dst=[194, 24, 460, 0, 0, 0, 0, 0] combine_rows_per_src=[8090, 12566, 7072, 0, 4358, 0, 0, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=158466048 dispatch_meta_bytes=77496 combine_payload_bytes=158466048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=158543544 ingress_bytes=1567752192 link_bytes=1567752192 token_bytes=317009592 weight_movement_bytes=1409286144 total_nvlink_bytes=1726295736 weight_to_token_ratio=4.445563 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.778757863e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7738, 7753, 0, 0, 811, 0, 82] payload_rows_per_dst=[5960, 9516, 0, 15506, 0, 1622, 0, 164] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5960, 9516, 0, 15506, 0, 1622, 0, 164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=171786240 dispatch_meta_bytes=84416 combine_payload_bytes=171786240 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=171870656 ingress_bytes=5104287744 link_bytes=5104287744 token_bytes=343656896 weight_movement_bytes=4932501504 total_nvlink_bytes=5276158400 weight_to_token_ratio=14.352983 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7808 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.858188073e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=159940608 dispatch_meta_bytes=80712 combine_payload_bytes=159940608 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=160021320 ingress_bytes=1569226752 link_bytes=1569226752 token_bytes=319961928 weight_movement_bytes=1409286144 total_nvlink_bytes=1729248072 weight_to_token_ratio=4.404543 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.787014784e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 6312, 7319, 0, 1, 2684, 0, 68] payload_rows_per_dst=[0, 8666, 3958, 0, 2, 5368, 14638, 136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8666, 3958, 0, 2, 5368, 14638, 136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19084 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 6841, 7525, 1, 4, 1897, 0, 115] payload_rows_per_dst=[13684, 0, 15050, 2, 8, 3794, 0, 230] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13684, 0, 15050, 2, 8, 3794, 0, 230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32532 payload_rows_remote=16820 dup_rows=236 dup_rows_remote=232 dedup_ratio=0.007202 router_tokens_per_expert=[0, 7840, 7763, 0, 0, 686, 0, 95] payload_rows_per_dst=[0, 0, 0, 0, 7746, 9074, 0, 15712] dup_rows_per_dst=[0, 0, 0, 0, 0, 232, 0, 4] combine_rows_per_src=[0, 0, 0, 0, 7746, 9074, 0, 15712] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=5905580032 link_bytes=5905580032 token_bytes=537001984 weight_movement_bytes=5637144576 total_nvlink_bytes=6174146560 weight_to_token_ratio=10.497437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6528 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306878307e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30068 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 7442, 7501, 2, 1, 1350, 0, 86] payload_rows_per_dst=[4, 0, 4036, 4, 10850, 2700, 14552, 622] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 4036, 4, 10850, 2700, 14552, 622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=156336128 dispatch_meta_bytes=76336 combine_payload_bytes=156336128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=156412464 ingress_bytes=1565622272 link_bytes=1565622272 token_bytes=312748592 weight_movement_bytes=1409286144 total_nvlink_bytes=1722034736 weight_to_token_ratio=4.506131 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.766831200e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=137789440 dispatch_meta_bytes=68208 combine_payload_bytes=137789440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=137857648 ingress_bytes=1547075584 link_bytes=1547075584 token_bytes=275647088 weight_movement_bytes=1409286144 total_nvlink_bytes=1684933232 weight_to_token_ratio=5.112647 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.662977489e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32758 payload_rows_remote=32614 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000305 router_tokens_per_expert=[0, 7080, 7762, 0, 1, 1450, 0, 91] payload_rows_per_dst=[0, 0, 0, 0, 14162, 2900, 144, 15552] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 10] combine_rows_per_src=[0, 0, 0, 0, 14162, 2900, 144, 15552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=246317056 dispatch_meta_bytes=120272 combine_payload_bytes=246317056 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=246437328 ingress_bytes=1655603200 link_bytes=1655603200 token_bytes=492754384 weight_movement_bytes=1409286144 total_nvlink_bytes=1902040528 weight_to_token_ratio=2.860017 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7040 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.270686837e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=20718 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 7293, 7742, 0, 0, 1259, 0, 90] payload_rows_per_dst=[0, 14586, 0, 12050, 0, 2518, 3434, 180] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 14586, 0, 12050, 0, 2518, 3434, 180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 6559, 7464, 1, 0, 2280, 0, 79] payload_rows_per_dst=[13120, 0, 9724, 5206, 0, 4560, 0, 158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13120, 0, 9724, 5206, 0, 4560, 0, 158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24640 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 14, 0, 8178, 8192, 0, 0] payload_rows_per_dst=[92, 8194, 8128, 8164, 0, 8190, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[92, 8194, 8128, 8164, 0, 8190, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=267173888 dispatch_meta_bytes=130496 combine_payload_bytes=267173888 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267304384 ingress_bytes=1676460032 link_bytes=1676460032 token_bytes=534478272 weight_movement_bytes=1409286144 total_nvlink_bytes=1943764416 weight_to_token_ratio=2.636751 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.387476390e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16416 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 69, 0, 8122, 8191, 1, 0] payload_rows_per_dst=[16352, 8190, 8224, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16352, 8190, 8224, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24634 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 138, 0, 8054, 8192, 0, 0] payload_rows_per_dst=[0, 0, 276, 7974, 8134, 8194, 8190, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 276, 7974, 8134, 8194, 8190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 5, 0, 8187, 8192, 0, 0] payload_rows_per_dst=[0, 0, 10, 0, 8190, 0, 8192, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 10, 0, 8190, 0, 8192, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=169721856 dispatch_meta_bytes=82872 combine_payload_bytes=169721856 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=169804728 ingress_bytes=1579008000 link_bytes=1579008000 token_bytes=339526584 weight_movement_bytes=1409286144 total_nvlink_bytes=1748812728 weight_to_token_ratio=4.150739 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.841785690e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=268566528 ingress_bytes=5905580032 link_bytes=5905580032 token_bytes=537001984 weight_movement_bytes=5637144576 total_nvlink_bytes=6174146560 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306878307e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201850880 dispatch_meta_bytes=98560 combine_payload_bytes=201850880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201949440 ingress_bytes=1611137024 link_bytes=1611137024 token_bytes=403800320 weight_movement_bytes=1409286144 total_nvlink_bytes=1813086464 weight_to_token_ratio=3.490057 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.021694813e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 0, 22, 2, 8168, 8190, 0, 0] payload_rows_per_dst=[4, 0, 44, 8170, 8034, 8188, 8192, 136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 44, 8170, 8034, 8188, 8192, 136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134479872 dispatch_meta_bytes=65664 combine_payload_bytes=134479872 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134545536 ingress_bytes=1543766016 link_bytes=1543766016 token_bytes=269025408 weight_movement_bytes=1409286144 total_nvlink_bytes=1678311552 weight_to_token_ratio=5.238487 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644445290e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201801728 dispatch_meta_bytes=98536 combine_payload_bytes=201801728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201900264 ingress_bytes=4429660160 link_bytes=4429660160 token_bytes=403701992 weight_movement_bytes=4227858432 total_nvlink_bytes=4631560424 weight_to_token_ratio=10.472721 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480424786e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 64, 0, 8128, 8192, 0, 0] payload_rows_per_dst=[0, 0, 128, 0, 8192, 2, 8190, 16256] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 128, 0, 8192, 2, 8190, 16256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98320 combine_payload_bytes=201359360 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201457680 ingress_bytes=4429217792 link_bytes=4429217792 token_bytes=402817040 weight_movement_bytes=4227858432 total_nvlink_bytes=4630675472 weight_to_token_ratio=10.495729 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480177079e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24562 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 18, 0, 8175, 8191, 0, 0] payload_rows_per_dst=[0, 8188, 7962, 8206, 218, 8194, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8188, 7962, 8206, 218, 8194, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 92, 0, 8099, 8191, 1, 0] payload_rows_per_dst=[16320, 8196, 7996, 254, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16320, 8196, 7996, 254, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16412 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[28, 4, 16356, 16380, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[28, 4, 16356, 16380, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16414 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 8191, 8176, 0, 15, 1] payload_rows_per_dst=[16354, 16382, 0, 0, 0, 0, 30, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16354, 16382, 0, 0, 0, 0, 30, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16414 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 30, 6, 16354, 16378, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 30, 6, 16354, 16378, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201211904 dispatch_meta_bytes=98248 combine_payload_bytes=201211904 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201310152 ingress_bytes=1610498048 link_bytes=1610498048 token_bytes=402522056 weight_movement_bytes=1409286144 total_nvlink_bytes=1811808200 weight_to_token_ratio=3.501140 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018116814e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201293824 dispatch_meta_bytes=98288 combine_payload_bytes=201293824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201392112 ingress_bytes=1610579968 link_bytes=1610579968 token_bytes=402685936 weight_movement_bytes=1409286144 total_nvlink_bytes=1811972080 weight_to_token_ratio=3.499715 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018575532e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 0, 0, 8190, 8188, 0, 2, 2] payload_rows_per_dst=[4, 0, 0, 0, 16376, 16380, 4, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 0, 0, 16376, 16380, 4, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134447104 dispatch_meta_bytes=65648 combine_payload_bytes=134447104 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134512752 ingress_bytes=1543733248 link_bytes=1543733248 token_bytes=268959856 weight_movement_bytes=1409286144 total_nvlink_bytes=1678246000 weight_to_token_ratio=5.239764 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644261803e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134463488 dispatch_meta_bytes=65656 combine_payload_bytes=134463488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134529144 ingress_bytes=1543749632 link_bytes=1543749632 token_bytes=268992632 weight_movement_bytes=1409286144 total_nvlink_bytes=1678278776 weight_to_token_ratio=5.239125 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644353546e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134463488 dispatch_meta_bytes=65656 combine_payload_bytes=134463488 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134529144 ingress_bytes=4362321920 link_bytes=4362321920 token_bytes=268992632 weight_movement_bytes=4227858432 total_nvlink_bytes=4496851064 weight_to_token_ratio=15.717376 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442718183e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16422 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 38, 8, 16346, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 38, 8, 16346, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8191, 8191, 1, 1, 0] payload_rows_per_dst=[0, 0, 16382, 16382, 0, 2, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16382, 16382, 0, 2, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 8191, 8190, 0, 1, 1] payload_rows_per_dst=[16382, 16382, 0, 0, 0, 0, 2, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 0, 0, 0, 0, 2, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134529024 dispatch_meta_bytes=65688 combine_payload_bytes=134529024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134594712 ingress_bytes=1543815168 link_bytes=1543815168 token_bytes=269123736 weight_movement_bytes=1409286144 total_nvlink_bytes=1678409880 weight_to_token_ratio=5.236573 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644720521e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 169, 0, 8192, 0, 8023, 0, 0] payload_rows_per_dst=[2240, 14144, 2, 16382, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2240, 14144, 2, 16382, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17228 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 422, 0, 8191, 0, 7769, 0, 1] payload_rows_per_dst=[15540, 844, 16382, 0, 0, 0, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[15540, 844, 16382, 0, 0, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16536 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 117, 0, 8192, 0, 8075, 0, 0] payload_rows_per_dst=[0, 234, 0, 0, 0, 8110, 8192, 16232] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 234, 0, 0, 0, 8110, 8192, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=17032 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 445, 0, 8192, 0, 7747, 0, 0] payload_rows_per_dst=[0, 890, 0, 2, 15736, 7950, 8190, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 890, 0, 2, 15736, 7950, 8190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=4362092544 link_bytes=4362092544 token_bytes=268533768 weight_movement_bytes=4227858432 total_nvlink_bytes=4496392200 weight_to_token_ratio=15.744234 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442589742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25416 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 378, 0, 8190, 2, 7812, 0, 0] payload_rows_per_dst=[4, 756, 0, 0, 15760, 7352, 8192, 704] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 756, 0, 0, 15760, 7352, 8192, 704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=141131776 dispatch_meta_bytes=68912 combine_payload_bytes=141131776 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=141200688 ingress_bytes=1550417920 link_bytes=1550417920 token_bytes=282332464 weight_movement_bytes=1409286144 total_nvlink_bytes=1691618608 weight_to_token_ratio=4.991584 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.681693176e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 335, 0, 8192, 0, 7857, 0, 0] payload_rows_per_dst=[0, 670, 0, 0, 0, 8074, 8194, 15830] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 670, 0, 0, 0, 8074, 8194, 15830] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=135462912 dispatch_meta_bytes=66144 combine_payload_bytes=135462912 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135529056 ingress_bytes=1544749056 link_bytes=1544749056 token_bytes=270991968 weight_movement_bytes=1409286144 total_nvlink_bytes=1680278112 weight_to_token_ratio=5.200472 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.649949904e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=139526144 dispatch_meta_bytes=68128 combine_payload_bytes=139526144 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=139594272 ingress_bytes=1548812288 link_bytes=1548812288 token_bytes=279120416 weight_movement_bytes=1409286144 total_nvlink_bytes=1688406560 weight_to_token_ratio=5.049026 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.672702307e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=208207872 dispatch_meta_bytes=101664 combine_payload_bytes=208207872 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=208309536 ingress_bytes=5140709376 link_bytes=5140709376 token_bytes=416517408 weight_movement_bytes=4932501504 total_nvlink_bytes=5349018912 weight_to_token_ratio=11.842246 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.878582667e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 371, 1, 8192, 0, 7820, 0, 0] payload_rows_per_dst=[0, 13830, 2, 16384, 1270, 1282, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 13830, 2, 16384, 1270, 1282, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=31368 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 700, 0, 8191, 1, 7491, 0, 0] payload_rows_per_dst=[14984, 1400, 16382, 0, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[14984, 1400, 16382, 0, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16656 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1, 0, 8191, 222, 86, 0, 7883] payload_rows_per_dst=[16112, 16040, 0, 0, 444, 172, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16112, 16040, 0, 0, 444, 172, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18354 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 125, 76, 0, 7991] payload_rows_per_dst=[648, 920, 14414, 16384, 250, 152, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[648, 920, 14414, 16384, 250, 152, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=31606 payload_rows_remote=15586 dup_rows=1162 dup_rows_remote=0 dedup_ratio=0.035461 router_tokens_per_expert=[0, 0, 0, 8192, 110, 78, 0, 8004] payload_rows_per_dst=[0, 0, 0, 0, 220, 156, 15210, 16020] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 1162] combine_rows_per_src=[0, 0, 0, 0, 220, 156, 15210, 16020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=134283264 ingress_bytes=5066719232 link_bytes=5066719232 token_bytes=268500992 weight_movement_bytes=4932501504 total_nvlink_bytes=5201002496 weight_to_token_ratio=18.370515 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.837151275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=256966656 dispatch_meta_bytes=125472 combine_payload_bytes=256966656 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=257092128 ingress_bytes=1666252800 link_bytes=1666252800 token_bytes=514058784 weight_movement_bytes=1409286144 total_nvlink_bytes=1923344928 weight_to_token_ratio=2.741488 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.330320152e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32622 payload_rows_remote=17404 dup_rows=146 dup_rows_remote=0 dedup_ratio=0.004456 router_tokens_per_expert=[2, 2, 1, 8190, 250, 79, 0, 7860] payload_rows_per_dst=[4, 4, 2, 0, 16204, 15218, 1174, 16] dup_rows_per_dst=[0, 0, 0, 0, 0, 146, 0, 0] combine_rows_per_src=[4, 4, 2, 0, 16204, 15218, 1174, 16] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32638 payload_rows_remote=18670 dup_rows=130 dup_rows_remote=130 dedup_ratio=0.003967 router_tokens_per_expert=[0, 0, 1, 8192, 379, 65, 0, 7747] payload_rows_per_dst=[0, 0, 2286, 0, 13968, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 130, 0, 0] combine_rows_per_src=[0, 0, 2286, 0, 13968, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 163, 76, 0, 7953] payload_rows_per_dst=[0, 0, 0, 0, 662, 152, 16384, 15570] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 662, 152, 16384, 15570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=136445952 dispatch_meta_bytes=66624 combine_payload_bytes=136445952 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=136512576 ingress_bytes=1545732096 link_bytes=1545732096 token_bytes=272958528 weight_movement_bytes=1409286144 total_nvlink_bytes=1682244672 weight_to_token_ratio=5.163005 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.655454517e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=150355968 dispatch_meta_bytes=73416 combine_payload_bytes=150355968 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=150429384 ingress_bytes=1559642112 link_bytes=1559642112 token_bytes=300785352 weight_movement_bytes=1409286144 total_nvlink_bytes=1710071496 weight_to_token_ratio=4.685355 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.733344801e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=127680512 dispatch_meta_bytes=62344 combine_payload_bytes=127680512 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=127742856 ingress_bytes=5764825088 link_bytes=5764825088 token_bytes=255423368 weight_movement_bytes=5637144576 total_nvlink_bytes=5892567944 weight_to_token_ratio=22.069808 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=7936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.228061414e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=142573568 dispatch_meta_bytes=69616 combine_payload_bytes=142573568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=142643184 ingress_bytes=1551859712 link_bytes=1551859712 token_bytes=285216752 weight_movement_bytes=1409286144 total_nvlink_bytes=1694502896 weight_to_token_ratio=4.941106 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.689766609e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=152944640 dispatch_meta_bytes=75200 combine_payload_bytes=152944640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=153019840 ingress_bytes=1562230784 link_bytes=1562230784 token_bytes=305964480 weight_movement_bytes=1409286144 total_nvlink_bytes=1715250624 weight_to_token_ratio=4.606045 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1024 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.747840283e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 8192, 86, 74, 0, 8030] payload_rows_per_dst=[0, 0, 16064, 16380, 172, 152, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16064, 16380, 172, 152, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16964 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1, 1, 8191, 424, 53, 0, 7713] payload_rows_per_dst=[16004, 15804, 2, 4, 848, 106, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16004, 15804, 2, 4, 848, 106, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 2, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16450 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8159, 32, 0, 0, 8191, 0, 1, 1] payload_rows_per_dst=[16318, 64, 16382, 0, 0, 0, 2, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16318, 64, 16382, 0, 0, 0, 2, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134316048 ingress_bytes=5771395072 link_bytes=5771395072 token_bytes=268566544 weight_movement_bytes=5637144576 total_nvlink_bytes=5905711120 weight_to_token_ratio=20.989750 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231740330e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 1, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[76, 16308, 4, 0, 16380, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[76, 16308, 4, 0, 16380, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=138969088 dispatch_meta_bytes=67856 combine_payload_bytes=138969088 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=139036944 ingress_bytes=1548255232 link_bytes=1548255232 token_bytes=278006032 weight_movement_bytes=1409286144 total_nvlink_bytes=1687292176 weight_to_token_ratio=5.069265 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.669583026e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8181, 9, 0, 0, 8190, 0, 2, 2] payload_rows_per_dst=[0, 18, 0, 16362, 0, 16380, 4, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 18, 0, 16362, 0, 16380, 4, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 2, 0, 16382, 4, 16380, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 16382, 4, 16380, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 0, 0, 8, 0, 8, 16376, 16376] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8, 0, 8, 16376, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134758400 dispatch_meta_bytes=65800 combine_payload_bytes=134758400 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134824200 ingress_bytes=4362616832 link_bytes=4362616832 token_bytes=269582600 weight_movement_bytes=4227858432 total_nvlink_bytes=4497441032 weight_to_token_ratio=15.682980 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442883321e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268533744 ingress_bytes=4496261120 link_bytes=4496261120 token_bytes=536936432 weight_movement_bytes=4227858432 total_nvlink_bytes=4764794864 weight_to_token_ratio=7.874039 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517718544e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 16368, 0, 16, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16368, 0, 16, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8187, 4, 0, 0, 8191, 0, 1, 1] payload_rows_per_dst=[16374, 8, 16382, 0, 0, 0, 2, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16374, 8, 16382, 0, 0, 0, 2, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268304384 dispatch_meta_bytes=131008 combine_payload_bytes=268304384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268435392 ingress_bytes=1677590528 link_bytes=1677590528 token_bytes=536739776 weight_movement_bytes=1409286144 total_nvlink_bytes=1946025920 weight_to_token_ratio=2.625641 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393806696e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 1, 8191, 8191, 0] payload_rows_per_dst=[16382, 16382, 2, 0, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 2, 0, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[4, 4, 16380, 0, 16380, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 4, 16380, 0, 16380, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 0, 2, 8190, 8190, 0] payload_rows_per_dst=[0, 0, 4, 16380, 4, 0, 16380, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 16380, 4, 0, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268369920 dispatch_meta_bytes=131040 combine_payload_bytes=268369920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268500960 ingress_bytes=1677656064 link_bytes=1677656064 token_bytes=536870880 weight_movement_bytes=1409286144 total_nvlink_bytes=1946157024 weight_to_token_ratio=2.625000 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394173670e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 16384, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[0, 0, 16380, 4, 16380, 0, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16380, 4, 16380, 0, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 1, 8191, 8191, 0] payload_rows_per_dst=[16382, 16382, 2, 0, 2, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 2, 0, 2, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8191, 1, 1, 0, 8191] payload_rows_per_dst=[16382, 16382, 0, 0, 2, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 0, 0, 2, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 8189, 2, 2, 0, 8190] payload_rows_per_dst=[0, 2, 0, 16378, 4, 4, 16380, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 16378, 4, 4, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[4, 0, 16384, 0, 0, 16380, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 16384, 0, 0, 16380, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 8191, 0, 0, 0, 8192] payload_rows_per_dst=[0, 2, 0, 16382, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 16382, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[0, 0, 0, 8, 16376, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8, 16376, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 8191, 0, 0, 0, 8192] payload_rows_per_dst=[0, 2, 16382, 0, 0, 16380, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 16382, 0, 0, 16380, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 0, 8189, 1, 1, 0, 8191] payload_rows_per_dst=[16382, 16380, 2, 0, 2, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16380, 2, 0, 2, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 8191, 8191, 1, 0] payload_rows_per_dst=[16382, 16382, 2, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 2, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 0, 8190, 8190, 2, 0] payload_rows_per_dst=[0, 0, 4, 16380, 0, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 16380, 0, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[2, 4, 16382, 0, 0, 16380, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 4, 16382, 0, 0, 16380, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 16384, 0, 4, 16380, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 4, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 16384, 0, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 16384, 0, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 0, 8191, 8192, 0, 0] payload_rows_per_dst=[2, 0, 16378, 4, 0, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 16378, 4, 0, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 0, 8191, 8191, 1, 0] payload_rows_per_dst=[16382, 16382, 2, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 2, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 3073, 0, 1, 5118, 0, 0] payload_rows_per_dst=[16384, 0, 10644, 0, 5740, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 10644, 0, 5740, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=31566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 8190, 3640, 0, 2, 4550, 0, 0] payload_rows_per_dst=[4, 0, 2448, 16380, 3634, 1202, 0, 9100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 2448, 16380, 3634, 1202, 0, 9100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=23490 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 3553, 0, 0, 4639, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 3666, 3440, 16384, 9278] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 3666, 3440, 16384, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=31014 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 1838, 0, 0, 6354, 0, 0] payload_rows_per_dst=[0, 0, 1922, 16384, 1754, 8802, 0, 3906] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 1922, 16384, 1754, 8802, 0, 3906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24844 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 4145, 0, 0, 4047, 0, 0] payload_rows_per_dst=[0, 16384, 7924, 0, 4816, 3644, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 7924, 0, 4816, 3644, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 2950, 0, 0, 5242, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 4008, 1892, 16384, 10484] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 4008, 1892, 16384, 10484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=258588672 dispatch_meta_bytes=126264 combine_payload_bytes=258588672 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=258714936 ingress_bytes=3781804032 link_bytes=3781804032 token_bytes=517303608 weight_movement_bytes=3523215360 total_nvlink_bytes=4040518968 weight_to_token_ratio=6.810730 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.117652398e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=192430080 dispatch_meta_bytes=93960 combine_payload_bytes=192430080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=192524040 ingress_bytes=1601716224 link_bytes=1601716224 token_bytes=384954120 weight_movement_bytes=1409286144 total_nvlink_bytes=1794240264 weight_to_token_ratio=3.660920 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.968942265e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=254066688 dispatch_meta_bytes=124056 combine_payload_bytes=254066688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=254190744 ingress_bytes=1663352832 link_bytes=1663352832 token_bytes=508257432 weight_movement_bytes=1409286144 total_nvlink_bytes=1917543576 weight_to_token_ratio=2.772780 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.314081542e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=203522048 dispatch_meta_bytes=99376 combine_payload_bytes=203522048 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=203621424 ingress_bytes=2317451264 link_bytes=2317451264 token_bytes=407143472 weight_movement_bytes=2113929216 total_nvlink_bytes=2521072688 weight_to_token_ratio=5.192099 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.297675973e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 2570, 0, 0, 5622, 0, 0] payload_rows_per_dst=[0, 16380, 2182, 4, 4594, 9608, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16380, 2182, 4, 4594, 9608, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 3170, 0, 1, 5021, 0, 0] payload_rows_per_dst=[16380, 4, 7648, 0, 4556, 4180, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 4, 7648, 0, 4556, 4180, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16514 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 1, 0, 0, 8191, 65, 8126] payload_rows_per_dst=[16254, 16382, 2, 0, 0, 0, 130, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16254, 16382, 2, 0, 0, 0, 130, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 0, 2, 0, 0, 8190, 67, 8123] payload_rows_per_dst=[4, 0, 4, 0, 16380, 0, 16204, 176] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 0, 4, 0, 16380, 0, 16204, 176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16542 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 79, 8113] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 158, 16226] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 158, 16226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268533744 ingress_bytes=4496261120 link_bytes=4496261120 token_bytes=536936432 weight_movement_bytes=4227858432 total_nvlink_bytes=4764794864 weight_to_token_ratio=7.874039 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517718544e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=135282688 dispatch_meta_bytes=66056 combine_payload_bytes=135282688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135348744 ingress_bytes=1544568832 link_bytes=1544568832 token_bytes=270631432 weight_movement_bytes=1409286144 total_nvlink_bytes=1679917576 weight_to_token_ratio=5.207400 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.648940724e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 49, 8143] payload_rows_per_dst=[0, 0, 0, 432, 16384, 0, 15952, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 432, 16384, 0, 15952, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 80, 8112] payload_rows_per_dst=[242, 4, 16380, 15982, 0, 0, 160, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[242, 4, 16380, 15982, 0, 0, 160, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 9, 8183] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 18, 16366] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 18, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=135512064 dispatch_meta_bytes=66168 combine_payload_bytes=135512064 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=135578232 ingress_bytes=4363370496 link_bytes=4363370496 token_bytes=271090296 weight_movement_bytes=4227858432 total_nvlink_bytes=4498948728 weight_to_token_ratio=15.595757 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.443305342e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16414 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 15, 8177] payload_rows_per_dst=[0, 0, 16380, 16354, 4, 0, 30, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16380, 16354, 4, 0, 30, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268288000 dispatch_meta_bytes=131000 combine_payload_bytes=268288000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268419000 ingress_bytes=1677574144 link_bytes=1677574144 token_bytes=536707000 weight_movement_bytes=1409286144 total_nvlink_bytes=1945993144 weight_to_token_ratio=2.625802 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393714952e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 1, 0, 0, 8191, 58, 8133] payload_rows_per_dst=[16268, 16382, 2, 0, 0, 0, 116, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16268, 16382, 2, 0, 0, 0, 116, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=16528 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[8111, 0, 8191, 1, 7, 73, 0, 1] payload_rows_per_dst=[16236, 8186, 8178, 2, 14, 146, 0, 2] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16236, 8186, 8178, 2, 14, 146, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134463488 dispatch_meta_bytes=65656 combine_payload_bytes=134463488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134529144 ingress_bytes=1543749632 link_bytes=1543749632 token_bytes=268992632 weight_movement_bytes=1409286144 total_nvlink_bytes=1678278776 weight_to_token_ratio=5.239125 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644353546e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24640 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8096, 1, 8190, 2, 0, 93, 0, 2] payload_rows_per_dst=[0, 2, 0, 8196, 8188, 8128, 8032, 222] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 8196, 8188, 8128, 8032, 222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16526 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8121, 0, 8192, 0, 0, 71, 0, 0] payload_rows_per_dst=[0, 0, 0, 0, 8192, 142, 8192, 16242] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 142, 8192, 16242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24586 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8100, 0, 8192, 0, 0, 92, 0, 0] payload_rows_per_dst=[0, 278, 6, 8196, 8182, 8018, 8088, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 278, 6, 8196, 8182, 8018, 8088, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=135397376 dispatch_meta_bytes=66112 combine_payload_bytes=135397376 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=135463488 ingress_bytes=5772541952 link_bytes=5772541952 token_bytes=270860864 weight_movement_bytes=5637144576 total_nvlink_bytes=5908005440 weight_to_token_ratio=20.811957 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.232382535e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8141, 0, 8192, 0, 0, 51, 0, 0] payload_rows_per_dst=[234, 7998, 8202, 8176, 0, 8158, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[234, 7998, 8202, 8176, 0, 8158, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8150, 0, 8192, 0, 1, 41, 0, 0] payload_rows_per_dst=[0, 0, 0, 4, 8190, 82, 8192, 16300] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 4, 8190, 82, 8192, 16300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 2, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201850880 dispatch_meta_bytes=98560 combine_payload_bytes=201850880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201949440 ingress_bytes=1611137024 link_bytes=1611137024 token_bytes=403800320 weight_movement_bytes=1409286144 total_nvlink_bytes=1813086464 weight_to_token_ratio=3.490057 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.021694813e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=135380992 dispatch_meta_bytes=66104 combine_payload_bytes=135380992 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135447096 ingress_bytes=1544667136 link_bytes=1544667136 token_bytes=270828088 weight_movement_bytes=1409286144 total_nvlink_bytes=1680114232 weight_to_token_ratio=5.203619 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.649491186e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201408512 dispatch_meta_bytes=98344 combine_payload_bytes=201408512 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201506856 ingress_bytes=1610694656 link_bytes=1610694656 token_bytes=402915368 weight_movement_bytes=1409286144 total_nvlink_bytes=1812201512 weight_to_token_ratio=3.497722 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019217737e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201244672 dispatch_meta_bytes=98264 combine_payload_bytes=201244672 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=201342936 ingress_bytes=5838389248 link_bytes=5838389248 token_bytes=402587608 weight_movement_bytes=5637144576 total_nvlink_bytes=6039732184 weight_to_token_ratio=14.002280 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269254273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8138, 0, 8192, 0, 0, 54, 0, 0] payload_rows_per_dst=[0, 8112, 8192, 8192, 0, 8008, 264, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8112, 8192, 8192, 0, 8008, 264, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8147, 0, 8191, 1, 1, 43, 0, 1] payload_rows_per_dst=[16294, 8192, 8190, 2, 2, 86, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16294, 8192, 8190, 2, 2, 86, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 3, 0, 1, 8189, 0] payload_rows_per_dst=[16382, 0, 16378, 6, 0, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 0, 16378, 6, 0, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8190, 0, 0, 11, 0, 2, 8181, 0] payload_rows_per_dst=[0, 0, 0, 22, 16380, 4, 16362, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 22, 16380, 4, 16362, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 0, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 4, 0, 0, 16380, 0, 16384, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 0, 0, 16380, 0, 16384, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=4362092544 link_bytes=4362092544 token_bytes=268533768 weight_movement_bytes=4227858432 total_nvlink_bytes=4496392200 weight_to_token_ratio=15.744234 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442589742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[4, 16380, 18, 16366, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 16380, 18, 16366, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 0, 0, 0, 8, 16376, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8, 16376, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268288000 dispatch_meta_bytes=131000 combine_payload_bytes=268288000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268419000 ingress_bytes=1677574144 link_bytes=1677574144 token_bytes=536707000 weight_movement_bytes=1409286144 total_nvlink_bytes=1945993144 weight_to_token_ratio=2.625802 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393714952e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16406 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 16384, 0, 16362, 0, 0, 22, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 16362, 0, 0, 22, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268566528 ingress_bytes=4496293888 link_bytes=4496293888 token_bytes=537001984 weight_movement_bytes=4227858432 total_nvlink_bytes=4764860416 weight_to_token_ratio=7.873078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517736893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 6, 0, 1, 8186, 0] payload_rows_per_dst=[16382, 0, 16372, 12, 0, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 0, 16372, 12, 0, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 0, 2, 1, 8189, 8191] payload_rows_per_dst=[16382, 16380, 0, 0, 4, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16380, 0, 0, 4, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134397952 dispatch_meta_bytes=65624 combine_payload_bytes=134397952 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134463576 ingress_bytes=1543684096 link_bytes=1543684096 token_bytes=268861528 weight_movement_bytes=1409286144 total_nvlink_bytes=1678147672 weight_to_token_ratio=5.241680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643986572e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 0, 0, 4, 2, 8186, 8190] payload_rows_per_dst=[0, 4, 0, 0, 16380, 16384, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 0, 0, 16380, 16384, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 4, 4, 16380, 16380, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 4, 16380, 16380, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[4, 4, 16380, 16380, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 4, 16380, 16380, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 0, 0, 0, 0, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 0, 0, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[0, 0, 16384, 16384, 0, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16384, 16384, 0, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134283264 ingress_bytes=4362076160 link_bytes=4362076160 token_bytes=268500992 weight_movement_bytes=4227858432 total_nvlink_bytes=4496359424 weight_to_token_ratio=15.746156 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442580568e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 0, 2, 1, 8189, 8191] payload_rows_per_dst=[16382, 16380, 0, 0, 4, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16380, 0, 0, 4, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16448 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 32, 0, 0, 8191, 0, 1, 8159] payload_rows_per_dst=[16320, 64, 16382, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16320, 64, 16382, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2, 4, 0, 0, 8190, 0, 2, 8186] payload_rows_per_dst=[4, 8, 0, 16302, 0, 16380, 4, 70] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4, 8, 0, 16302, 0, 16380, 4, 70] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134742016 dispatch_meta_bytes=65792 combine_payload_bytes=134742016 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134807808 ingress_bytes=1544028160 link_bytes=1544028160 token_bytes=269549824 weight_movement_bytes=1409286144 total_nvlink_bytes=1678835968 weight_to_token_ratio=5.228296 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645913187e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 18, 0, 0, 8192, 0, 0, 8174] payload_rows_per_dst=[0, 36, 0, 16348, 4, 16380, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 36, 0, 16348, 4, 16380, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 18, 0, 0, 8192, 0, 0, 8174] payload_rows_per_dst=[84, 16300, 4, 0, 16380, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[84, 16300, 4, 0, 16380, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16414 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 15, 0, 0, 8192, 0, 0, 8177] payload_rows_per_dst=[0, 30, 0, 0, 0, 0, 16384, 16354] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 30, 0, 0, 0, 0, 16384, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 20, 0, 0, 8192, 0, 0, 8172] payload_rows_per_dst=[0, 40, 0, 0, 0, 8, 16376, 16344] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 40, 0, 0, 0, 8, 16376, 16344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4440656368 prefetch_bytes=19730006016 reduce_bytes=12683575296 weight_movement_bytes=32413581312 egress_bytes=2220870128 ingress_bytes=34633367552 link_bytes=34633367552 total_nvlink_bytes=36854237680 weight_to_token_ratio=7.299277 prefetch_share_of_ingress=0.569682 dedup_ratio=0.000500 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=18 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020360 gemm_inner_pad_upper_bound=7239 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.212077510e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 10, 0, 0, 8192, 0, 0, 8182] payload_rows_per_dst=[0, 16266, 0, 118, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16266, 0, 118, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268533744 ingress_bytes=4496261120 link_bytes=4496261120 token_bytes=536936432 weight_movement_bytes=4227858432 total_nvlink_bytes=4764794864 weight_to_token_ratio=7.874039 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517718544e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32744 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 12, 0, 0, 8191, 0, 1, 8179] payload_rows_per_dst=[16360, 24, 16382, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16360, 24, 16382, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134463488 dispatch_meta_bytes=65656 combine_payload_bytes=134463488 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134529144 ingress_bytes=4362321920 link_bytes=4362321920 token_bytes=268992632 weight_movement_bytes=4227858432 total_nvlink_bytes=4496851064 weight_to_token_ratio=15.717376 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442718183e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6716818792 prefetch_bytes=14092861440 reduce_bytes=29595009024 weight_movement_bytes=43687870464 egress_bytes=3359229288 ingress_bytes=47045459968 link_bytes=47045459968 total_nvlink_bytes=50404689256 weight_to_token_ratio=6.504250 prefetch_share_of_ingress=0.299558 dedup_ratio=0.000439 prefetch_slots=40 unused_prefetch_slots=24 slot_utilization=0.625000 reduce_slots=42 remote_demand=20 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023674 gemm_inner_pad_upper_bound=7493 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.646468363e-04 prefetch_bytes_per_gemm_flop=4.932133838e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6655527856 prefetch_bytes=15502147584 reduce_bytes=21139292160 weight_movement_bytes=36641439744 egress_bytes=3328576432 ingress_bytes=39968391168 link_bytes=39968391168 total_nvlink_bytes=43296967600 weight_to_token_ratio=5.505415 prefetch_share_of_ingress=0.387860 dedup_ratio=0.000248 prefetch_slots=44 unused_prefetch_slots=20 slot_utilization=0.687500 reduce_slots=30 remote_demand=22 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.022254 gemm_inner_pad_upper_bound=7493 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.398789418e-04 prefetch_bytes_per_gemm_flop=5.425347222e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=267468800 dispatch_meta_bytes=130600 combine_payload_bytes=267468800 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267599400 ingress_bytes=1676754944 link_bytes=1676754944 token_bytes=535068200 weight_movement_bytes=1409286144 total_nvlink_bytes=1944354344 weight_to_token_ratio=2.633844 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.389127774e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6204961624 prefetch_bytes=20434649088 reduce_bytes=11978932224 weight_movement_bytes=32413581312 egress_bytes=3103241048 ingress_bytes=35515301888 link_bytes=35515301888 total_nvlink_bytes=38618542936 weight_to_token_ratio=5.223817 prefetch_share_of_ingress=0.575376 dedup_ratio=0.003822 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=17 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023201 gemm_inner_pad_upper_bound=6477 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.242942911e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=268238848 dispatch_meta_bytes=130976 combine_payload_bytes=268238848 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268369824 ingress_bytes=1677524992 link_bytes=1677524992 token_bytes=536608672 weight_movement_bytes=1409286144 total_nvlink_bytes=1945894816 weight_to_token_ratio=2.626283 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393439722e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4501784704 prefetch_bytes=16911433728 reduce_bytes=21139292160 weight_movement_bytes=38050725888 egress_bytes=2251442304 ingress_bytes=40301068288 link_bytes=40301068288 total_nvlink_bytes=42552510592 weight_to_token_ratio=8.452365 prefetch_share_of_ingress=0.419627 dedup_ratio=0.002750 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=30 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.024621 gemm_inner_pad_upper_bound=6731 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.410432249e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6356348168 prefetch_bytes=18320719872 reduce_bytes=12683575296 weight_movement_bytes=31004295168 egress_bytes=3178949896 ingress_bytes=34181693440 link_bytes=34181693440 total_nvlink_bytes=37360643336 weight_to_token_ratio=4.877690 prefetch_share_of_ingress=0.535980 dedup_ratio=0.000076 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=18 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.024621 gemm_inner_pad_upper_bound=6350 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.196270095e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6320032736 prefetch_bytes=16911433728 reduce_bytes=22548578304 weight_movement_bytes=39460012032 egress_bytes=3160787936 ingress_bytes=42619256832 link_bytes=42619256832 total_nvlink_bytes=45780044768 weight_to_token_ratio=6.243640 prefetch_share_of_ingress=0.396803 dedup_ratio=0.000256 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=32 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.026042 gemm_inner_pad_upper_bound=6223 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.491562801e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6213152280 prefetch_bytes=19730006016 reduce_bytes=9865003008 weight_movement_bytes=29595009024 egress_bytes=3107335704 ingress_bytes=32700825600 link_bytes=32700825600 total_nvlink_bytes=35808161304 weight_to_token_ratio=4.763284 prefetch_share_of_ingress=0.603349 dedup_ratio=0.001293 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=14 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023201 gemm_inner_pad_upper_bound=6477 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.144443584e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none - [2026-09-16 21:34:45] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 586.8 | throughput per GPU (TFLOP/s/GPU): 568.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.387856E+00 | loss scale: 1.0 | grad norm: 5.073 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32128 payload_rows_remote=32128 dup_rows=640 dup_rows_remote=640 dedup_ratio=0.019531 router_tokens_per_expert=[973, 3, 3951, 5932, 637, 4886, 1, 1] payload_rows_per_dst=[1946, 6, 0, 5442, 6602, 0, 8168, 9964] dup_rows_per_dst=[0, 0, 0, 528, 112, 0, 0, 0] combine_rows_per_src=[1946, 6, 0, 5442, 6602, 0, 8168, 9964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32502 payload_rows_remote=21336 dup_rows=266 dup_rows_remote=266 dedup_ratio=0.008118 router_tokens_per_expert=[2012, 6, 3190, 5984, 840, 4351, 1, 0] payload_rows_per_dst=[4024, 12, 0, 0, 7796, 0, 9504, 11166] dup_rows_per_dst=[0, 0, 0, 0, 264, 0, 2, 0] combine_rows_per_src=[4024, 12, 0, 0, 7796, 0, 9504, 11166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32134 payload_rows_remote=19886 dup_rows=634 dup_rows_remote=256 dedup_ratio=0.019348 router_tokens_per_expert=[574, 1, 1783, 6690, 422, 6914, 0, 0] payload_rows_per_dst=[12248, 13380, 4942, 676, 844, 44, 0, 0] dup_rows_per_dst=[378, 2, 254, 0, 0, 0, 0, 0] combine_rows_per_src=[12248, 13380, 4942, 676, 844, 44, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25798 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2495, 16, 3485, 5013, 1597, 3777, 0, 1] payload_rows_per_dst=[4990, 7198, 6970, 2860, 3194, 7554, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4990, 7198, 6970, 2860, 3194, 7554, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=20542 dup_rows=96 dup_rows_remote=94 dedup_ratio=0.002930 router_tokens_per_expert=[800, 10, 3939, 6056, 778, 4794, 3, 4] payload_rows_per_dst=[1706, 12130, 8078, 0, 1556, 9188, 6, 8] dup_rows_per_dst=[12, 2, 82, 0, 0, 0, 0, 0] combine_rows_per_src=[1706, 12130, 8078, 0, 1556, 9188, 6, 8] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32502 payload_rows_remote=29170 dup_rows=266 dup_rows_remote=208 dedup_ratio=0.008118 router_tokens_per_expert=[1327, 15, 3678, 5379, 732, 5250, 1, 2] payload_rows_per_dst=[2654, 30, 4916, 11064, 3332, 6186, 2874, 1446] dup_rows_per_dst=[0, 0, 0, 208, 58, 0, 0, 0] combine_rows_per_src=[2654, 30, 4916, 11064, 3332, 6186, 2874, 1446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=263192576 dispatch_meta_bytes=131072 combine_payload_bytes=263192576 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=263323648 ingress_bytes=5195694080 link_bytes=5195694080 token_bytes=526516224 weight_movement_bytes=4932501504 total_nvlink_bytes=5459017728 weight_to_token_ratio=9.368185 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.909371806e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=174784512 dispatch_meta_bytes=86408 combine_payload_bytes=174784512 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=174870920 ingress_bytes=1584070656 link_bytes=1584070656 token_bytes=349655432 weight_movement_bytes=1409286144 total_nvlink_bytes=1758941576 weight_to_token_ratio=4.030500 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.870134450e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=162906112 dispatch_meta_bytes=80568 combine_payload_bytes=162906112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=162986680 ingress_bytes=1572192256 link_bytes=1572192256 token_bytes=325892792 weight_movement_bytes=1409286144 total_nvlink_bytes=1735178936 weight_to_token_ratio=4.324386 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5632 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.803620368e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=211337216 dispatch_meta_bytes=103192 combine_payload_bytes=211337216 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=211440408 ingress_bytes=3734552576 link_bytes=3734552576 token_bytes=422777624 weight_movement_bytes=3523215360 total_nvlink_bytes=3945992984 weight_to_token_ratio=8.333495 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.091193555e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32640 payload_rows_remote=20428 dup_rows=128 dup_rows_remote=128 dedup_ratio=0.003906 router_tokens_per_expert=[1195, 0, 3394, 6216, 599, 4979, 1, 0] payload_rows_per_dst=[2390, 0, 0, 0, 7858, 0, 12212, 10180] dup_rows_per_dst=[0, 0, 0, 0, 128, 0, 0, 0] combine_rows_per_src=[2390, 0, 0, 0, 7858, 0, 12212, 10180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=20778 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1210, 4, 3763, 5995, 512, 4898, 1, 1] payload_rows_per_dst=[2420, 8, 7526, 11990, 1024, 9796, 2, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2420, 8, 7526, 11990, 1024, 9796, 2, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=168280064 dispatch_meta_bytes=82544 combine_payload_bytes=168280064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=168362608 ingress_bytes=1577566208 link_bytes=1577566208 token_bytes=336642672 weight_movement_bytes=1409286144 total_nvlink_bytes=1745928816 weight_to_token_ratio=4.186297 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.833712256e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=238960640 dispatch_meta_bytes=117512 combine_payload_bytes=238960640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=239078152 ingress_bytes=1648246784 link_bytes=1648246784 token_bytes=478038792 weight_movement_bytes=1409286144 total_nvlink_bytes=1887324936 weight_to_token_ratio=2.948058 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3840 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.229493978e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=167346176 dispatch_meta_bytes=82224 combine_payload_bytes=167346176 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=167428400 ingress_bytes=1576632320 link_bytes=1576632320 token_bytes=334774576 weight_movement_bytes=1409286144 total_nvlink_bytes=1744060720 weight_to_token_ratio=4.209657 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.828482873e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=170213376 dispatch_meta_bytes=83112 combine_payload_bytes=170213376 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=170296488 ingress_bytes=4398071808 link_bytes=4398071808 token_bytes=340509864 weight_movement_bytes=4227858432 total_nvlink_bytes=4568368296 weight_to_token_ratio=12.416258 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.462736628e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24258 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 2765, 368, 1, 51, 7635, 0, 5563] payload_rows_per_dst=[8510, 6900, 11862, 5394, 102, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8510, 6900, 11862, 5394, 102, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30148 payload_rows_remote=21386 dup_rows=2620 dup_rows_remote=748 dedup_ratio=0.079956 router_tokens_per_expert=[1, 6109, 905, 1, 374, 6205, 0, 2789] payload_rows_per_dst=[2, 0, 1810, 2, 12218, 0, 7354, 8762] dup_rows_per_dst=[0, 0, 0, 0, 748, 0, 0, 1872] combine_rows_per_src=[2, 0, 1810, 2, 12218, 0, 7354, 8762] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=30482 payload_rows_remote=23338 dup_rows=2286 dup_rows_remote=2284 dedup_ratio=0.069763 router_tokens_per_expert=[6, 6022, 1000, 4, 634, 5721, 0, 2997] payload_rows_per_dst=[8334, 6182, 7144, 4706, 1268, 2000, 0, 848] dup_rows_per_dst=[4, 2280, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8334, 6182, 7144, 4706, 1268, 2000, 0, 848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32588 payload_rows_remote=29560 dup_rows=180 dup_rows_remote=180 dedup_ratio=0.005493 router_tokens_per_expert=[0, 5733, 422, 1, 208, 7219, 0, 2801] payload_rows_per_dst=[0, 3088, 844, 4384, 4232, 3028, 11410, 5602] dup_rows_per_dst=[0, 0, 0, 2, 178, 0, 0, 0] combine_rows_per_src=[0, 3088, 844, 4384, 4232, 3028, 11410, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=198721536 dispatch_meta_bytes=97032 combine_payload_bytes=198721536 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=198818568 ingress_bytes=1608007680 link_bytes=1608007680 token_bytes=397540104 weight_movement_bytes=1409286144 total_nvlink_bytes=1806826248 weight_to_token_ratio=3.545016 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.004171793e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32328 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 5340, 509, 0, 220, 6880, 0, 3435] payload_rows_per_dst=[0, 4974, 1018, 5706, 440, 13760, 0, 6870] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4974, 1018, 5706, 440, 13760, 0, 6870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32760 payload_rows_remote=29090 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000244 router_tokens_per_expert=[2, 5427, 404, 3, 173, 7395, 0, 2980] payload_rows_per_dst=[15018, 3670, 6760, 6966, 346, 0, 0, 0] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[15018, 3670, 6760, 6966, 346, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=175194112 dispatch_meta_bytes=88536 combine_payload_bytes=175194112 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=175282648 ingress_bytes=2993766400 link_bytes=2993766400 token_bytes=350476760 weight_movement_bytes=2818572288 total_nvlink_bytes=3169049048 weight_to_token_ratio=8.042109 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=7168 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.676384218e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=191184896 dispatch_meta_bytes=102488 combine_payload_bytes=191184896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=191287384 ingress_bytes=1600471040 link_bytes=1600471040 token_bytes=382472280 weight_movement_bytes=1409286144 total_nvlink_bytes=1791758424 weight_to_token_ratio=3.684675 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2560 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.961969755e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27162 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 6090, 625, 0, 265, 6990, 0, 2414] payload_rows_per_dst=[900, 5674, 2258, 5606, 530, 13980, 0, 3820] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[900, 5674, 2258, 5606, 530, 13980, 0, 3820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=242155520 dispatch_meta_bytes=118960 combine_payload_bytes=242155520 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=242274480 ingress_bytes=5879300096 link_bytes=5879300096 token_bytes=484430000 weight_movement_bytes=5637144576 total_nvlink_bytes=6121574576 weight_to_token_ratio=11.636655 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.292162640e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32366 payload_rows_remote=18362 dup_rows=402 dup_rows_remote=402 dedup_ratio=0.012268 router_tokens_per_expert=[0, 6152, 531, 1, 201, 7002, 0, 2497] payload_rows_per_dst=[0, 0, 1062, 2, 12304, 0, 14004, 4994] dup_rows_per_dst=[0, 0, 0, 0, 402, 0, 0, 0] combine_rows_per_src=[0, 0, 1062, 2, 12304, 0, 14004, 4994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=264830976 dispatch_meta_bytes=129312 combine_payload_bytes=264830976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=264960288 ingress_bytes=1674117120 link_bytes=1674117120 token_bytes=529791264 weight_movement_bytes=1409286144 total_nvlink_bytes=1939077408 weight_to_token_ratio=2.660078 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=1152 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.374357061e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=238305280 dispatch_meta_bytes=116392 combine_payload_bytes=238305280 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=238421672 ingress_bytes=4466163712 link_bytes=4466163712 token_bytes=476726952 weight_movement_bytes=4227858432 total_nvlink_bytes=4704585384 weight_to_token_ratio=8.868511 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.500865252e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19400 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 5950, 0, 2241, 8191, 1, 0] payload_rows_per_dst=[13368, 8192, 11206, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13368, 8192, 11206, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=222511104 dispatch_meta_bytes=108648 combine_payload_bytes=222511104 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=222619752 ingress_bytes=1631797248 link_bytes=1631797248 token_bytes=445130856 weight_movement_bytes=1409286144 total_nvlink_bytes=1854417000 weight_to_token_ratio=3.166004 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.137383443e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=150421504 dispatch_meta_bytes=75056 combine_payload_bytes=150421504 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=150496560 ingress_bytes=1559707648 link_bytes=1559707648 token_bytes=300918064 weight_movement_bytes=1409286144 total_nvlink_bytes=1710204208 weight_to_token_ratio=4.683289 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.733711775e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7, 0, 2339, 5, 5848, 8184, 0, 1] payload_rows_per_dst=[2604, 8196, 2088, 8182, 11696, 0, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2604, 8196, 2088, 8182, 11696, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=158924800 dispatch_meta_bytes=77600 combine_payload_bytes=158924800 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=159002400 ingress_bytes=1568210944 link_bytes=1568210944 token_bytes=317927200 weight_movement_bytes=1409286144 total_nvlink_bytes=1727213344 weight_to_token_ratio=4.432732 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.781326683e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3190, 0, 5002, 8192, 0, 0] payload_rows_per_dst=[622, 0, 2248, 8198, 7084, 8188, 3510, 2918] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[622, 0, 2248, 8198, 7084, 8188, 3510, 2918] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=22220 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 2917, 0, 5274, 8191, 1, 0] payload_rows_per_dst=[2, 0, 2266, 0, 0, 8190, 11762, 10548] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 2266, 0, 0, 8190, 11762, 10548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=24568 dup_rows=6 dup_rows_remote=0 dedup_ratio=0.000183 router_tokens_per_expert=[1, 0, 3252, 0, 4940, 8191, 0, 0] payload_rows_per_dst=[2, 0, 2744, 8188, 18, 8194, 3760, 9856] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[2, 0, 2744, 8188, 18, 8194, 3760, 9856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3176, 0, 5016, 8192, 0, 0] payload_rows_per_dst=[3870, 8188, 724, 8196, 10032, 0, 1758, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3870, 8188, 724, 8196, 10032, 0, 1758, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=251330560 dispatch_meta_bytes=122720 combine_payload_bytes=251330560 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=251453280 ingress_bytes=3069902848 link_bytes=3069902848 token_bytes=502783840 weight_movement_bytes=2818572288 total_nvlink_bytes=3321356128 weight_to_token_ratio=5.605933 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6272 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.719017451e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=210403328 dispatch_meta_bytes=102736 combine_payload_bytes=210403328 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=210506064 ingress_bytes=4438261760 link_bytes=4438261760 token_bytes=420909392 weight_movement_bytes=4227858432 total_nvlink_bytes=4648767824 weight_to_token_ratio=10.044581 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.485241323e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=182026240 dispatch_meta_bytes=88880 combine_payload_bytes=182026240 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=182115120 ingress_bytes=1591312384 link_bytes=1591312384 token_bytes=364141360 weight_movement_bytes=1409286144 total_nvlink_bytes=1773427504 weight_to_token_ratio=3.870162 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.910685104e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=20794 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 3469, 0, 4722, 8191, 1, 0] payload_rows_per_dst=[2, 0, 3158, 0, 0, 8190, 11974, 9444] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 3158, 0, 0, 8190, 11974, 9444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=22450 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 8191, 2533, 4, 5654, 1] payload_rows_per_dst=[10318, 8190, 0, 0, 2942, 11316, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10318, 8190, 0, 0, 2942, 11316, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201261056 dispatch_meta_bytes=98272 combine_payload_bytes=201261056 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=201359328 ingress_bytes=5838405632 link_bytes=5838405632 token_bytes=402620384 weight_movement_bytes=5637144576 total_nvlink_bytes=6039764960 weight_to_token_ratio=14.001141 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269263447e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201293824 dispatch_meta_bytes=98288 combine_payload_bytes=201293824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201392112 ingress_bytes=1610579968 link_bytes=1610579968 token_bytes=402685936 weight_movement_bytes=1409286144 total_nvlink_bytes=1811972080 weight_to_token_ratio=3.499715 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018575532e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 0, 3447, 2, 4742, 8189, 1, 0] payload_rows_per_dst=[12298, 8192, 8334, 4, 3938, 0, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12298, 8192, 8334, 4, 3938, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=170344448 dispatch_meta_bytes=83176 combine_payload_bytes=170344448 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=170427624 ingress_bytes=1579630592 link_bytes=1579630592 token_bytes=340772072 weight_movement_bytes=1409286144 total_nvlink_bytes=1750058216 weight_to_token_ratio=4.135568 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.845271945e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=183910400 dispatch_meta_bytes=89800 combine_payload_bytes=183910400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=184000200 ingress_bytes=1593196544 link_bytes=1593196544 token_bytes=367910600 weight_movement_bytes=1409286144 total_nvlink_bytes=1777196744 weight_to_token_ratio=3.830512 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.921235613e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7, 4, 0, 8184, 6301, 0, 1885, 3] payload_rows_per_dst=[6904, 8188, 8188, 0, 5712, 3770, 0, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6904, 8188, 8188, 0, 5712, 3770, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27738 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8191, 4802, 2, 3389, 0] payload_rows_per_dst=[0, 0, 8194, 8188, 5030, 3426, 6164, 1766] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8194, 8188, 5030, 3426, 6164, 1766] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18502 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 8191, 5262, 8, 2921, 1] payload_rows_per_dst=[2, 0, 0, 8188, 1478, 16, 8818, 14266] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 8188, 1478, 16, 8818, 14266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32742 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8191, 3814, 13, 4366, 0] payload_rows_per_dst=[0, 0, 8188, 8194, 3598, 26, 8732, 4030] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 8188, 8194, 3598, 26, 8732, 4030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=32764 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 8191, 4907, 12, 3274, 0] payload_rows_per_dst=[2356, 8186, 8196, 2, 6156, 6570, 1300, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[2356, 8186, 8196, 2, 6156, 6570, 1300, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201359360 dispatch_meta_bytes=98320 combine_payload_bytes=201359360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201457680 ingress_bytes=1610645504 link_bytes=1610645504 token_bytes=402817040 weight_movement_bytes=1409286144 total_nvlink_bytes=1812103184 weight_to_token_ratio=3.498576 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018942506e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=227229696 dispatch_meta_bytes=110952 combine_payload_bytes=227229696 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=227340648 ingress_bytes=4455088128 link_bytes=4455088128 token_bytes=454570344 weight_movement_bytes=4227858432 total_nvlink_bytes=4682428776 weight_to_token_ratio=9.300779 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.494663387e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 2, 2, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 360, 8191, 1, 7233, 598, 0] payload_rows_per_dst=[16384, 7818, 5528, 0, 2, 1840, 1196, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 7818, 5528, 0, 2, 1840, 1196, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25014 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 8191, 4309, 5, 3877, 1] payload_rows_per_dst=[0, 0, 2, 8196, 4114, 10, 7754, 12692] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 8196, 4114, 10, 7754, 12692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=151568384 dispatch_meta_bytes=74008 combine_payload_bytes=151568384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=151642392 ingress_bytes=1560854528 link_bytes=1560854528 token_bytes=303210776 weight_movement_bytes=1409286144 total_nvlink_bytes=1712496920 weight_to_token_ratio=4.647876 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.740133824e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=268222464 dispatch_meta_bytes=130968 combine_payload_bytes=268222464 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268353432 ingress_bytes=1677508608 link_bytes=1677508608 token_bytes=536575896 weight_movement_bytes=1409286144 total_nvlink_bytes=1945862040 weight_to_token_ratio=2.626443 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393347978e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131064 combine_payload_bytes=268402688 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=268533752 ingress_bytes=5905547264 link_bytes=5905547264 token_bytes=536936440 weight_movement_bytes=5637144576 total_nvlink_bytes=6174081016 weight_to_token_ratio=10.498719 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306859958e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 0, 0, 8189, 4373, 2, 3814, 3] payload_rows_per_dst=[13188, 8204, 0, 0, 3738, 7632, 0, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[13188, 8204, 0, 0, 3738, 7632, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=204914688 dispatch_meta_bytes=100056 combine_payload_bytes=204914688 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=205014744 ingress_bytes=3023486976 link_bytes=3023486976 token_bytes=409929432 weight_movement_bytes=2818572288 total_nvlink_bytes=3228501720 weight_to_token_ratio=6.875750 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6784 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.693026500e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=29208 payload_rows_remote=20370 dup_rows=3560 dup_rows_remote=754 dedup_ratio=0.108643 router_tokens_per_expert=[1, 37, 2914, 8184, 1, 5034, 207, 6] payload_rows_per_dst=[2, 10222, 8838, 3326, 2, 6392, 414, 12] dup_rows_per_dst=[0, 58, 2806, 696, 0, 0, 0, 0] combine_rows_per_src=[2, 10222, 8838, 3326, 2, 6392, 414, 12] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=201228288 dispatch_meta_bytes=98256 combine_payload_bytes=201228288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201326544 ingress_bytes=1610514432 link_bytes=1610514432 token_bytes=402554832 weight_movement_bytes=1409286144 total_nvlink_bytes=1811840976 weight_to_token_ratio=3.500855 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018208557e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28746 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 1417, 8192, 0, 5997, 777, 0] payload_rows_per_dst=[0, 2, 2834, 12362, 4022, 6082, 7466, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 2834, 12362, 4022, 6082, 7466, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29116 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8, 942, 8191, 1, 6521, 721, 0] payload_rows_per_dst=[0, 16, 1884, 8140, 8244, 3652, 2658, 8174] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16, 1884, 8140, 8244, 3652, 2658, 8174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32250 payload_rows_remote=20702 dup_rows=518 dup_rows_remote=518 dedup_ratio=0.015808 router_tokens_per_expert=[1, 0, 1957, 8191, 0, 5773, 461, 1] payload_rows_per_dst=[2, 0, 3914, 0, 8138, 54, 8594, 11548] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 518, 0] combine_rows_per_src=[2, 0, 3914, 0, 8138, 54, 8594, 11548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24524 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1628, 8192, 0, 5731, 833, 0] payload_rows_per_dst=[0, 8140, 3256, 8244, 0, 7378, 5750, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8140, 3256, 8244, 0, 7378, 5750, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=166871040 dispatch_meta_bytes=84496 combine_payload_bytes=166871040 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=166955536 ingress_bytes=1576157184 link_bytes=1576157184 token_bytes=333826576 weight_movement_bytes=1409286144 total_nvlink_bytes=1743112720 weight_to_token_ratio=4.221612 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5888 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.825822310e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19828 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 0, 11, 6469, 1711, 1, 8191] payload_rows_per_dst=[12940, 8190, 8192, 22, 0, 3422, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12940, 8190, 8192, 22, 0, 3422, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 1, 1, 0, 2, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235487232 dispatch_meta_bytes=114984 combine_payload_bytes=235487232 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235602216 ingress_bytes=1644773376 link_bytes=1644773376 token_bytes=471089448 weight_movement_bytes=1409286144 total_nvlink_bytes=1880375592 weight_to_token_ratio=2.991547 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210044343e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32520 payload_rows_remote=27410 dup_rows=248 dup_rows_remote=0 dedup_ratio=0.007568 router_tokens_per_expert=[1, 0, 1009, 8191, 0, 6515, 667, 1] payload_rows_per_dst=[2, 0, 2018, 0, 12358, 0, 5110, 13032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 248, 0] combine_rows_per_src=[2, 0, 2018, 0, 12358, 0, 5110, 13032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=238518272 dispatch_meta_bytes=116464 combine_payload_bytes=238518272 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=238634736 ingress_bytes=5171019776 link_bytes=5171019776 token_bytes=477153008 weight_movement_bytes=4932501504 total_nvlink_bytes=5409654512 weight_to_token_ratio=10.337358 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.895555226e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=169590784 dispatch_meta_bytes=84880 combine_payload_bytes=169590784 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=169675664 ingress_bytes=1578876928 link_bytes=1578876928 token_bytes=339266448 weight_movement_bytes=1409286144 total_nvlink_bytes=1748552592 weight_to_token_ratio=4.153921 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.841051741e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=200900608 dispatch_meta_bytes=98096 combine_payload_bytes=200900608 prefetch_bytes=704643072 reduce_bytes=5637144576 egress_bytes=200998704 ingress_bytes=6542688256 link_bytes=6542688256 token_bytes=401899312 weight_movement_bytes=6341787648 total_nvlink_bytes=6743686960 weight_to_token_ratio=15.779543 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.663632318e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26256 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 3, 843, 8189, 1, 6937, 407, 1] payload_rows_per_dst=[16378, 6512, 1690, 0, 2, 7370, 814, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 6512, 1690, 0, 2, 7370, 814, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=162430976 dispatch_meta_bytes=79312 combine_payload_bytes=162430976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=162510288 ingress_bytes=1571717120 link_bytes=1571717120 token_bytes=324941264 weight_movement_bytes=1409286144 total_nvlink_bytes=1734227408 weight_to_token_ratio=4.337049 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.800959805e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=224542720 dispatch_meta_bytes=109640 combine_payload_bytes=224542720 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=224652360 ingress_bytes=1633828864 link_bytes=1633828864 token_bytes=449195080 weight_movement_bytes=1409286144 total_nvlink_bytes=1858481224 weight_to_token_ratio=3.137359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=2688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.148759645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=28598 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000183 router_tokens_per_expert=[7, 1, 0, 3, 6794, 1387, 8, 8184] payload_rows_per_dst=[5570, 3872, 4164, 16366, 0, 2774, 16, 0] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[5570, 3872, 4164, 16366, 0, 2774, 16, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=215089152 dispatch_meta_bytes=105024 combine_payload_bytes=215089152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=215194176 ingress_bytes=1624375296 link_bytes=1624375296 token_bytes=430283328 weight_movement_bytes=1409286144 total_nvlink_bytes=1839569472 weight_to_token_ratio=3.275252 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.095823610e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30378 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 2, 6484, 1706, 0, 8191] payload_rows_per_dst=[0, 5522, 5072, 4, 2390, 3412, 16368, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 5522, 5072, 4, 2390, 3412, 16368, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=29990 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 3, 6800, 1389, 0, 8190] payload_rows_per_dst=[0, 0, 4, 6, 13600, 2778, 16376, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4, 6, 13600, 2778, 16376, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114416 combine_payload_bytes=234274816 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234389232 ingress_bytes=1643560960 link_bytes=1643560960 token_bytes=468664048 weight_movement_bytes=1409286144 total_nvlink_bytes=1877950192 weight_to_token_ratio=3.007028 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203255320e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 1, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1, 0, 8, 6755, 1428, 0, 8191] payload_rows_per_dst=[2, 2, 0, 16, 3062, 13304, 0, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 2, 0, 16, 3062, 13304, 0, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32752 payload_rows_remote=16440 dup_rows=16 dup_rows_remote=0 dedup_ratio=0.000488 router_tokens_per_expert=[0, 0, 0, 8, 7067, 1117, 0, 8192] payload_rows_per_dst=[0, 6986, 7148, 16312, 66, 2234, 6, 0] dup_rows_per_dst=[0, 0, 0, 16, 0, 0, 0, 0] combine_rows_per_src=[0, 6986, 7148, 16312, 66, 2234, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32758 payload_rows_remote=20118 dup_rows=10 dup_rows_remote=0 dedup_ratio=0.000305 router_tokens_per_expert=[6319, 1872, 1, 0, 8191, 0, 0, 1] payload_rows_per_dst=[12640, 3744, 8190, 8182, 0, 0, 0, 2] dup_rows_per_dst=[10, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12640, 3744, 8190, 8182, 0, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=248856576 dispatch_meta_bytes=121512 combine_payload_bytes=248856576 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=248978088 ingress_bytes=5886001152 link_bytes=5886001152 token_bytes=497834664 weight_movement_bytes=5637144576 total_nvlink_bytes=6134979240 weight_to_token_ratio=11.323327 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.295914951e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=245678080 dispatch_meta_bytes=119960 combine_payload_bytes=245678080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=245798040 ingress_bytes=1654964224 link_bytes=1654964224 token_bytes=491476120 weight_movement_bytes=1409286144 total_nvlink_bytes=1900762264 weight_to_token_ratio=2.867456 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5760 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.267108838e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 1, 0, 0, 6825, 1366, 0, 8191] payload_rows_per_dst=[2, 2, 0, 0, 13650, 2732, 0, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 2, 0, 0, 13650, 2732, 0, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 3, 0, 9, 7124, 1056, 0, 8189] payload_rows_per_dst=[14254, 8194, 8188, 20, 0, 2112, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[14254, 8194, 8188, 20, 0, 2112, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=134299656 ingress_bytes=5771378688 link_bytes=5771378688 token_bytes=268533768 weight_movement_bytes=5637144576 total_nvlink_bytes=5905678344 weight_to_token_ratio=20.992312 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.231731156e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=134676480 dispatch_meta_bytes=65760 combine_payload_bytes=134676480 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134742240 ingress_bytes=1543962624 link_bytes=1543962624 token_bytes=269418720 weight_movement_bytes=1409286144 total_nvlink_bytes=1678704864 weight_to_token_ratio=5.230840 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.645546213e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=164806656 dispatch_meta_bytes=80472 combine_payload_bytes=164806656 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=164887128 ingress_bytes=5801951232 link_bytes=5801951232 token_bytes=329693784 weight_movement_bytes=5637144576 total_nvlink_bytes=5966838360 weight_to_token_ratio=17.098122 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.248850505e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32764 payload_rows_remote=32750 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[7609, 570, 5, 5, 8184, 0, 3, 8] payload_rows_per_dst=[5794, 10564, 14, 8198, 8172, 0, 6, 16] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[5794, 10564, 14, 8198, 8172, 0, 6, 16] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7400, 792, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 1584, 7538, 4, 8208, 15434, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 1584, 7538, 4, 8208, 15434, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19256 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7426, 766, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 1532, 6646, 0, 8192, 13512, 2886, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 1532, 6646, 0, 8192, 13512, 2886, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=268288000 dispatch_meta_bytes=131016 combine_payload_bytes=268288000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268419016 ingress_bytes=1677574144 link_bytes=1677574144 token_bytes=536707016 weight_movement_bytes=1409286144 total_nvlink_bytes=1945993160 weight_to_token_ratio=2.625802 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393714952e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 2, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7575, 616, 0, 0, 8191, 0, 1, 1] payload_rows_per_dst=[0, 1232, 0, 0, 0, 0, 15152, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 1232, 0, 0, 0, 0, 15152, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7501, 691, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[0, 10392, 2190, 8192, 8192, 3802, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 10392, 2190, 8192, 8192, 3802, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201195520 dispatch_meta_bytes=98240 combine_payload_bytes=201195520 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=201293760 ingress_bytes=5838340096 link_bytes=5838340096 token_bytes=402489280 weight_movement_bytes=5637144576 total_nvlink_bytes=6039633856 weight_to_token_ratio=14.005701 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.269226749e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 3602, 1, 8191, 0, 4589] payload_rows_per_dst=[16382, 0, 2, 11432, 4952, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 0, 2, 11432, 4952, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=157745152 dispatch_meta_bytes=77024 combine_payload_bytes=157745152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=157822176 ingress_bytes=1567031296 link_bytes=1567031296 token_bytes=315567328 weight_movement_bytes=1409286144 total_nvlink_bytes=1724853472 weight_to_token_ratio=4.465881 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.774721147e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18050 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7358, 833, 0, 0, 8191, 0, 1, 1] payload_rows_per_dst=[0, 1666, 0, 0, 4, 20, 14718, 16360] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 1666, 0, 0, 4, 20, 14718, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=30714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[7162, 1027, 0, 0, 8189, 0, 3, 3] payload_rows_per_dst=[14324, 2054, 8190, 8188, 0, 0, 6, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[14324, 2054, 8190, 8188, 0, 0, 6, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8, 4109, 8, 8184, 0, 4075] payload_rows_per_dst=[8, 16360, 16, 3628, 10882, 0, 0, 1874] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 16360, 16, 3628, 10882, 0, 0, 1874] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=147865600 dispatch_meta_bytes=72200 combine_payload_bytes=147865600 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=147937800 ingress_bytes=1557151744 link_bytes=1557151744 token_bytes=295803400 weight_movement_bytes=1409286144 total_nvlink_bytes=1705089544 weight_to_token_ratio=4.764266 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.719399779e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=251609088 dispatch_meta_bytes=122856 combine_payload_bytes=251609088 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=251731944 ingress_bytes=1660895232 link_bytes=1660895232 token_bytes=503341032 weight_movement_bytes=1409286144 total_nvlink_bytes=1912627176 weight_to_token_ratio=2.799863 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=3712 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.300320007e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25468 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 5505, 0, 8192, 0, 2687] payload_rows_per_dst=[0, 24, 16360, 0, 7300, 0, 5342, 3742] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 24, 16360, 0, 7300, 0, 5342, 3742] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 5955, 0, 8192, 0, 2237] payload_rows_per_dst=[0, 0, 16380, 0, 0, 4, 11910, 4474] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 16380, 0, 0, 4, 11910, 4474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268304384 dispatch_meta_bytes=131008 combine_payload_bytes=268304384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268435392 ingress_bytes=1677590528 link_bytes=1677590528 token_bytes=536739776 weight_movement_bytes=1409286144 total_nvlink_bytes=1946025920 weight_to_token_ratio=2.625641 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393806696e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=20314 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 3925, 1, 8191, 0, 4266] payload_rows_per_dst=[0, 0, 2, 0, 2, 16382, 3928, 12454] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 2, 16382, 3928, 12454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28086 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 5358, 0, 8192, 0, 2834] payload_rows_per_dst=[0, 16384, 0, 4682, 6272, 0, 1636, 3794] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16384, 0, 4682, 6272, 0, 1636, 3794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=208633856 dispatch_meta_bytes=101872 combine_payload_bytes=208633856 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=208735728 ingress_bytes=1617920000 link_bytes=1617920000 token_bytes=417369584 weight_movement_bytes=1409286144 total_nvlink_bytes=1826655728 weight_to_token_ratio=3.376590 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.059676647e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268533744 ingress_bytes=4496261120 link_bytes=4496261120 token_bytes=536936432 weight_movement_bytes=4227858432 total_nvlink_bytes=4764794864 weight_to_token_ratio=7.874039 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517718544e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=22816 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 1, 5751, 1, 8191, 0, 2440] payload_rows_per_dst=[0, 0, 2, 0, 2, 16382, 9952, 6430] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 2, 16382, 9952, 6430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 8190, 1, 1, 0, 8191] payload_rows_per_dst=[16382, 16382, 0, 0, 2, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16382, 0, 0, 2, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 3, 5622, 3, 8189, 0, 2567] payload_rows_per_dst=[16378, 0, 6, 13026, 3358, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 0, 6, 13026, 3358, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=230080512 dispatch_meta_bytes=112344 combine_payload_bytes=230080512 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=230192856 ingress_bytes=3753295872 link_bytes=3753295872 token_bytes=460273368 weight_movement_bytes=3523215360 total_nvlink_bytes=3983488728 weight_to_token_ratio=7.654615 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.101689018e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=166412288 dispatch_meta_bytes=81256 combine_payload_bytes=166412288 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=166493544 ingress_bytes=2280341504 link_bytes=2280341504 token_bytes=332905832 weight_movement_bytes=2113929216 total_nvlink_bytes=2446835048 weight_to_token_ratio=6.349931 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.276896056e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16404 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8184, 8, 8, 0, 8184] payload_rows_per_dst=[8, 0, 16364, 8176, 16, 8204, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 0, 16364, 8176, 16, 8204, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=186908672 dispatch_meta_bytes=91264 combine_payload_bytes=186908672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=186999936 ingress_bytes=1596194816 link_bytes=1596194816 token_bytes=373908608 weight_movement_bytes=1409286144 total_nvlink_bytes=1783194752 weight_to_token_ratio=3.769066 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.938024685e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 8191, 0, 0, 0, 8192] payload_rows_per_dst=[0, 2, 18, 8208, 8174, 8174, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 18, 8208, 8174, 8174, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2, 1, 8190, 0, 0, 0, 8191] payload_rows_per_dst=[0, 4, 2, 8190, 8190, 8190, 8188, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 4, 2, 8190, 8190, 8190, 8188, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134381568 dispatch_meta_bytes=65616 combine_payload_bytes=134381568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134447184 ingress_bytes=1543667712 link_bytes=1543667712 token_bytes=268828752 weight_movement_bytes=1409286144 total_nvlink_bytes=1678114896 weight_to_token_ratio=5.242319 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643894829e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 8191, 0, 0, 0, 8192] payload_rows_per_dst=[0, 2, 16384, 8190, 0, 8188, 4, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 16384, 8190, 0, 8188, 4, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201474048 dispatch_meta_bytes=98376 combine_payload_bytes=201474048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201572424 ingress_bytes=1610760192 link_bytes=1610760192 token_bytes=403046472 weight_movement_bytes=1409286144 total_nvlink_bytes=1812332616 weight_to_token_ratio=3.496585 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019584711e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201441288 ingress_bytes=1610629120 link_bytes=1610629120 token_bytes=402784264 weight_movement_bytes=1409286144 total_nvlink_bytes=1812070408 weight_to_token_ratio=3.498861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018850762e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8191, 1, 1, 0, 8191] payload_rows_per_dst=[0, 0, 0, 0, 8192, 2, 8192, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 0, 8192, 2, 8192, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8191, 1, 1, 0, 8191] payload_rows_per_dst=[0, 0, 0, 4, 8188, 2, 8192, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 4, 8188, 2, 8192, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28440 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2164, 0, 1, 0, 1084, 8191, 1, 4943] payload_rows_per_dst=[4328, 16382, 2, 0, 2168, 0, 9888, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4328, 16382, 2, 0, 2168, 0, 9888, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8189, 3, 3, 0, 8189] payload_rows_per_dst=[16378, 16378, 0, 0, 6, 6, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 16378, 0, 0, 6, 6, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201342976 dispatch_meta_bytes=98312 combine_payload_bytes=201342976 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201441288 ingress_bytes=4429201408 link_bytes=4429201408 token_bytes=402784264 weight_movement_bytes=4227858432 total_nvlink_bytes=4630642696 weight_to_token_ratio=10.496583 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480167905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5249, 0, 8, 0, 747, 8184, 8, 2188] payload_rows_per_dst=[10498, 8, 16376, 0, 1494, 0, 4392, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10498, 8, 16376, 0, 1494, 0, 4392, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=4362092544 link_bytes=4362092544 token_bytes=268533768 weight_movement_bytes=4227858432 total_nvlink_bytes=4496392200 weight_to_token_ratio=15.744234 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442589742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=232980480 dispatch_meta_bytes=113760 combine_payload_bytes=232980480 prefetch_bytes=0 reduce_bytes=2818572288 egress_bytes=233094240 ingress_bytes=3051552768 link_bytes=3051552768 token_bytes=466074720 weight_movement_bytes=2818572288 total_nvlink_bytes=3284647008 weight_to_token_ratio=6.047469 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708742172e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28466 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3208, 0, 0, 0, 1471, 8192, 0, 3513] payload_rows_per_dst=[5056, 0, 0, 16384, 4302, 0, 3416, 3610] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[5056, 0, 0, 16384, 4302, 0, 3416, 3610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2520, 0, 0, 0, 1618, 8192, 0, 4054] payload_rows_per_dst=[1354, 0, 0, 16380, 6922, 4, 0, 8108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[1354, 0, 0, 16380, 6922, 4, 0, 8108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3354, 0, 0, 0, 1688, 8192, 0, 3150] payload_rows_per_dst=[6708, 0, 16380, 4, 3376, 0, 6300, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[6708, 0, 16380, 4, 3376, 0, 6300, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=233193472 dispatch_meta_bytes=113864 combine_payload_bytes=233193472 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233307336 ingress_bytes=1642479616 link_bytes=1642479616 token_bytes=466500808 weight_movement_bytes=1409286144 total_nvlink_bytes=1875786952 weight_to_token_ratio=3.020973 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=5632 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.197200245e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=268533744 ingress_bytes=4496261120 link_bytes=4496261120 token_bytes=536936432 weight_movement_bytes=4227858432 total_nvlink_bytes=4764794864 weight_to_token_ratio=7.874039 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.517718544e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=19410 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4152, 0, 1, 0, 1080, 8191, 1, 2959] payload_rows_per_dst=[0, 0, 2, 0, 3024, 16382, 2, 13358] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 3024, 16382, 2, 13358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2543, 0, 1, 0, 1985, 8191, 1, 3663] payload_rows_per_dst=[0, 0, 2, 0, 8690, 16382, 2, 7692] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 0, 8690, 16382, 2, 7692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 8165, 0, 1, 26, 0, 0] payload_rows_per_dst=[16384, 0, 16330, 0, 2, 52, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16330, 0, 2, 52, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2412, 0, 3, 0, 1396, 8189, 3, 4381] payload_rows_per_dst=[4824, 16378, 6, 0, 2792, 0, 8768, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4824, 16378, 6, 0, 2792, 0, 8768, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32718 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8, 8184, 8177, 4, 8, 3, 0, 0] payload_rows_per_dst=[16, 16368, 50, 8, 16320, 6, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16, 16368, 50, 8, 16320, 6, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=159006720 dispatch_meta_bytes=77640 combine_payload_bytes=159006720 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=159084360 ingress_bytes=2977579008 link_bytes=2977579008 token_bytes=318091080 weight_movement_bytes=2818572288 total_nvlink_bytes=3136663368 weight_to_token_ratio=8.860897 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=6528 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.667319954e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=268419072 dispatch_meta_bytes=131064 combine_payload_bytes=268419072 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268550136 ingress_bytes=1677705216 link_bytes=1677705216 token_bytes=536969208 weight_movement_bytes=1409286144 total_nvlink_bytes=1946255352 weight_to_token_ratio=2.624519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394448901e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32708 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8191, 0, 0, 0, 1, 0] payload_rows_per_dst=[0, 0, 12, 16372, 60, 16322, 2, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 12, 16372, 60, 16322, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16390 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8191, 0, 0, 0, 1, 0] payload_rows_per_dst=[0, 0, 0, 16384, 0, 16378, 2, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 0, 16378, 2, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268025856 dispatch_meta_bytes=130872 combine_payload_bytes=268025856 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=268156728 ingress_bytes=5905170432 link_bytes=5905170432 token_bytes=536182584 weight_movement_bytes=5637144576 total_nvlink_bytes=6173327160 weight_to_token_ratio=10.513479 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306648948e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8188, 0, 0, 1, 3, 0] payload_rows_per_dst=[0, 16380, 4, 0, 16376, 2, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16380, 4, 0, 16376, 2, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=267943936 dispatch_meta_bytes=130832 combine_payload_bytes=267943936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268074768 ingress_bytes=1677230080 link_bytes=1677230080 token_bytes=536018704 weight_movement_bytes=1409286144 total_nvlink_bytes=1945304848 weight_to_token_ratio=2.629173 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.391788338e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134266880 dispatch_meta_bytes=65560 combine_payload_bytes=134266880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134332440 ingress_bytes=1543553024 link_bytes=1543553024 token_bytes=268599320 weight_movement_bytes=1409286144 total_nvlink_bytes=1677885464 weight_to_token_ratio=5.246797 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643252624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 8191, 0, 1, 0, 0, 0] payload_rows_per_dst=[2, 0, 0, 0, 2, 0, 16382, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 0, 2, 0, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16392 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8191, 8191, 0, 1, 0, 0, 0] payload_rows_per_dst=[2, 0, 0, 4, 2, 2, 16376, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 4, 2, 2, 16376, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 1, 1, 0, 8191, 16, 8174] payload_rows_per_dst=[16384, 0, 16350, 2, 0, 0, 32, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 0, 16350, 2, 0, 0, 32, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268435456 dispatch_meta_bytes=131072 combine_payload_bytes=268435456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268566528 ingress_bytes=1677721600 link_bytes=1677721600 token_bytes=537001984 weight_movement_bytes=1409286144 total_nvlink_bytes=1946288128 weight_to_token_ratio=2.624359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394540645e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32748 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 8189, 8186, 0, 3, 3, 0, 0] payload_rows_per_dst=[16364, 20, 16372, 0, 6, 6, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16364, 20, 16372, 0, 6, 6, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32626 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8, 0, 9, 28, 4, 8184, 34, 8117] payload_rows_per_dst=[16, 16368, 142, 56, 16118, 0, 68, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16, 16368, 142, 56, 16118, 0, 68, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=134283264 dispatch_meta_bytes=65568 combine_payload_bytes=134283264 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134348832 ingress_bytes=1543569408 link_bytes=1543569408 token_bytes=268632096 weight_movement_bytes=1409286144 total_nvlink_bytes=1677918240 weight_to_token_ratio=5.246157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643344367e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32472 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 9, 8183] payload_rows_per_dst=[0, 0, 0, 16384, 296, 0, 16088, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16384, 296, 0, 16088, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 4, 8188] payload_rows_per_dst=[0, 0, 0, 16314, 0, 70, 16288, 96] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 16314, 0, 70, 16288, 96] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=268271616 dispatch_meta_bytes=130992 combine_payload_bytes=268271616 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=268402608 ingress_bytes=5905416192 link_bytes=5905416192 token_bytes=536674224 weight_movement_bytes=5637144576 total_nvlink_bytes=6173818800 weight_to_token_ratio=10.503848 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306786563e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=267272192 dispatch_meta_bytes=130504 combine_payload_bytes=267272192 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=267402696 ingress_bytes=1676558336 link_bytes=1676558336 token_bytes=534674888 weight_movement_bytes=1409286144 total_nvlink_bytes=1943961032 weight_to_token_ratio=2.635781 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.388026852e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 15, 8177] payload_rows_per_dst=[0, 16380, 0, 4, 16354, 0, 30, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 16380, 0, 4, 16354, 0, 30, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=266010624 dispatch_meta_bytes=129888 combine_payload_bytes=266010624 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=266140512 ingress_bytes=1675296768 link_bytes=1675296768 token_bytes=532151136 weight_movement_bytes=1409286144 total_nvlink_bytes=1941437280 weight_to_token_ratio=2.648282 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.380962598e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=267862016 dispatch_meta_bytes=130792 combine_payload_bytes=267862016 prefetch_bytes=0 reduce_bytes=5637144576 egress_bytes=267992808 ingress_bytes=5905006592 link_bytes=5905006592 token_bytes=535854824 weight_movement_bytes=5637144576 total_nvlink_bytes=6172999400 weight_to_token_ratio=10.519910 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.306557204e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32758 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 0, 1, 2, 0, 8191, 5, 8184] payload_rows_per_dst=[2, 0, 2, 4, 0, 16382, 10, 16368] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 2, 4, 0, 16382, 10, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=16464 dup_rows=66 dup_rows_remote=0 dedup_ratio=0.002014 router_tokens_per_expert=[1, 0, 1, 0, 0, 8191, 72, 8119] payload_rows_per_dst=[2, 0, 2, 0, 0, 16316, 144, 16238] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 66] combine_rows_per_src=[2, 0, 2, 0, 0, 16316, 144, 16238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16388 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4, 8186, 8191, 1, 1, 0, 0, 1] payload_rows_per_dst=[16380, 4, 16378, 2, 2, 0, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 4, 16378, 2, 2, 0, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268402688 dispatch_meta_bytes=131056 combine_payload_bytes=268402688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268533744 ingress_bytes=1677688832 link_bytes=1677688832 token_bytes=536936432 weight_movement_bytes=1409286144 total_nvlink_bytes=1946222576 weight_to_token_ratio=2.624680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394357157e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268353536 dispatch_meta_bytes=131032 combine_payload_bytes=268353536 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268484568 ingress_bytes=1677639680 link_bytes=1677639680 token_bytes=536838104 weight_movement_bytes=1409286144 total_nvlink_bytes=1946124248 weight_to_token_ratio=2.625160 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.394081927e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4, 8180, 8182, 8, 0, 0, 0, 10] payload_rows_per_dst=[8, 16360, 12, 16, 16352, 0, 0, 20] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 16360, 12, 16, 16352, 0, 0, 20] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[5, 8187, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[10, 6, 0, 16368, 28, 8168, 8188, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[10, 6, 0, 16368, 28, 8168, 8188, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32748 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 0, 3, 2, 0, 8189, 54, 8133] payload_rows_per_dst=[16364, 20, 16272, 4, 0, 0, 108, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16364, 20, 16272, 4, 0, 0, 108, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=134873088 dispatch_meta_bytes=65856 combine_payload_bytes=134873088 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=134938944 ingress_bytes=5772017664 link_bytes=5772017664 token_bytes=269812032 weight_movement_bytes=5637144576 total_nvlink_bytes=5906956608 weight_to_token_ratio=20.892858 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.232088956e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134250496 dispatch_meta_bytes=65552 combine_payload_bytes=134250496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134316048 ingress_bytes=1543536640 link_bytes=1543536640 token_bytes=268566544 weight_movement_bytes=1409286144 total_nvlink_bytes=1677852688 weight_to_token_ratio=5.247437 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643160880e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[4, 8188, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[8, 0, 0, 16352, 0, 8204, 8204, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 0, 0, 16352, 0, 8204, 8204, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32748 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[6, 8186, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[12, 16352, 0, 20, 16384, 0, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[12, 16352, 0, 20, 16384, 0, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268337152 dispatch_meta_bytes=131024 combine_payload_bytes=268337152 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=268468176 ingress_bytes=5200838656 link_bytes=5200838656 token_bytes=536805328 weight_movement_bytes=4932501504 total_nvlink_bytes=5469306832 weight_to_token_ratio=9.188622 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912252554e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268206080 dispatch_meta_bytes=130960 combine_payload_bytes=268206080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268337040 ingress_bytes=1677492224 link_bytes=1677492224 token_bytes=536543120 weight_movement_bytes=1409286144 total_nvlink_bytes=1945829264 weight_to_token_ratio=2.626604 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393256235e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=268271616 dispatch_meta_bytes=130992 combine_payload_bytes=268271616 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268402608 ingress_bytes=1677557760 link_bytes=1677557760 token_bytes=536674224 weight_movement_bytes=1409286144 total_nvlink_bytes=1945960368 weight_to_token_ratio=2.625962 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393623209e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201228288 dispatch_meta_bytes=98256 combine_payload_bytes=201228288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201326544 ingress_bytes=1610514432 link_bytes=1610514432 token_bytes=402554832 weight_movement_bytes=1409286144 total_nvlink_bytes=1811840976 weight_to_token_ratio=3.500855 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018208557e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8190, 8191, 1, 0, 0, 0, 1] payload_rows_per_dst=[2, 0, 0, 2, 0, 8214, 8194, 16356] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 2, 0, 8214, 8194, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16384 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8, 8182, 8191, 1, 1, 0, 0, 1] payload_rows_per_dst=[16, 0, 0, 2, 2, 8182, 8182, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16, 0, 0, 2, 2, 8182, 8182, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 42, 0, 1, 8150, 0] payload_rows_per_dst=[16382, 8192, 8108, 84, 0, 2, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 8192, 8108, 84, 0, 2, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268271616 dispatch_meta_bytes=130992 combine_payload_bytes=268271616 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=268402608 ingress_bytes=1677557760 link_bytes=1677557760 token_bytes=536674224 weight_movement_bytes=1409286144 total_nvlink_bytes=1945960368 weight_to_token_ratio=2.625962 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.393623209e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=201310208 dispatch_meta_bytes=98296 combine_payload_bytes=201310208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201408504 ingress_bytes=1610596352 link_bytes=1610596352 token_bytes=402718712 weight_movement_bytes=1409286144 total_nvlink_bytes=1812004856 weight_to_token_ratio=3.499431 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018667275e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[0, 2, 22, 0, 16382, 8170, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 22, 0, 16382, 8170, 8192, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24406 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8183, 0, 0, 308, 0, 8, 7884, 1] payload_rows_per_dst=[8, 8378, 8362, 16002, 0, 16, 0, 2] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8, 8378, 8362, 16002, 0, 16, 0, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=32722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 8188, 8189, 3, 0, 0, 0, 3] payload_rows_per_dst=[16332, 46, 16378, 6, 0, 0, 0, 6] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16332, 46, 16378, 6, 0, 0, 0, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65536 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283264 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268500992 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787136 weight_to_token_ratio=5.248719 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=134299656 ingress_bytes=4362092544 link_bytes=4362092544 token_bytes=268533768 weight_movement_bytes=4227858432 total_nvlink_bytes=4496392200 weight_to_token_ratio=15.744234 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.442589742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 1, 0, 0, 0, 0, 8191, 0] payload_rows_per_dst=[0, 2, 0, 0, 16384, 8192, 8006, 184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 2, 0, 0, 16384, 8192, 8006, 184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 2, 0, 0, 0, 0, 8190, 0] payload_rows_per_dst=[0, 8196, 8192, 16194, 0, 0, 186, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 8196, 8192, 16194, 0, 0, 186, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=199933952 dispatch_meta_bytes=97624 combine_payload_bytes=199933952 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=200031576 ingress_bytes=1609220096 link_bytes=1609220096 token_bytes=399965528 weight_movement_bytes=1409286144 total_nvlink_bytes=1809251672 weight_to_token_ratio=3.523519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.010960816e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=268058624 dispatch_meta_bytes=130888 combine_payload_bytes=268058624 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=268189512 ingress_bytes=5200560128 link_bytes=5200560128 token_bytes=536248136 weight_movement_bytes=4932501504 total_nvlink_bytes=5468749640 weight_to_token_ratio=9.198170 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.912096590e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 1, 53, 0, 0, 8139, 0] payload_rows_per_dst=[0, 0, 2, 106, 2, 8190, 8192, 16276] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 2, 106, 2, 8190, 8192, 16276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201424896 ingress_bytes=1610612736 link_bytes=1610612736 token_bytes=402751488 weight_movement_bytes=1409286144 total_nvlink_bytes=1812037632 weight_to_token_ratio=3.499146 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.018759019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16462 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8191, 0, 0, 39, 0, 1, 8153, 0] payload_rows_per_dst=[0, 0, 0, 78, 0, 8192, 8192, 16306] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 78, 0, 8192, 8192, 16306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=135774208 dispatch_meta_bytes=66296 combine_payload_bytes=135774208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=135840504 ingress_bytes=1545060352 link_bytes=1545060352 token_bytes=271614712 weight_movement_bytes=1409286144 total_nvlink_bytes=1680900856 weight_to_token_ratio=5.188549 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.651693031e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16386 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 1, 0, 6859, 11, 1, 1321, 8191] payload_rows_per_dst=[16382, 6474, 7246, 0, 22, 2, 2642, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 6474, 7246, 0, 22, 2, 2642, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=201326592 dispatch_meta_bytes=98304 combine_payload_bytes=201326592 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=201424896 ingress_bytes=4429185024 link_bytes=4429185024 token_bytes=402751488 weight_movement_bytes=4227858432 total_nvlink_bytes=4630609920 weight_to_token_ratio=10.497437 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.480158730e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=24584 dup_rows=76 dup_rows_remote=76 dedup_ratio=0.002319 router_tokens_per_expert=[0, 0, 0, 5843, 0, 0, 2349, 8192] payload_rows_per_dst=[0, 0, 4908, 6786, 8108, 8192, 4698, 0] dup_rows_per_dst=[0, 0, 0, 76, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 4908, 6786, 8108, 8192, 4698, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=134856704 dispatch_meta_bytes=65848 combine_payload_bytes=134856704 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134922552 ingress_bytes=1544142848 link_bytes=1544142848 token_bytes=269779256 weight_movement_bytes=1409286144 total_nvlink_bytes=1679065400 weight_to_token_ratio=5.223849 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.646555392e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=26730 dup_rows=98 dup_rows_remote=98 dedup_ratio=0.002991 router_tokens_per_expert=[0, 9, 0, 7332, 52, 8, 800, 8183] payload_rows_per_dst=[8, 12790, 5940, 4140, 8176, 16, 1600, 0] dup_rows_per_dst=[0, 0, 0, 0, 98, 0, 0, 0] combine_rows_per_src=[8, 12790, 5940, 4140, 8176, 16, 1600, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=24770 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8189, 0, 0, 152, 0, 3, 8040, 0] payload_rows_per_dst=[16378, 7998, 8082, 304, 0, 6, 0, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 7998, 8082, 304, 0, 6, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=134234112 dispatch_meta_bytes=65544 combine_payload_bytes=134234112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134299656 ingress_bytes=1543520256 link_bytes=1543520256 token_bytes=268533768 weight_movement_bytes=1409286144 total_nvlink_bytes=1677819912 weight_to_token_ratio=5.248078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.643069137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32732 payload_rows_remote=19914 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 5312, 0, 0, 2880, 8192] payload_rows_per_dst=[0, 0, 2296, 3564, 8188, 12818, 5862, 4] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 36, 0] combine_rows_per_src=[0, 0, 2296, 3564, 8188, 12818, 5862, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=201392128 dispatch_meta_bytes=98640 combine_payload_bytes=201392128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=201490768 ingress_bytes=1610678272 link_bytes=1610678272 token_bytes=402882896 weight_movement_bytes=1409286144 total_nvlink_bytes=1812169040 weight_to_token_ratio=3.498004 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019125993e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=27724 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 6038, 0, 0, 2154, 8192] payload_rows_per_dst=[0, 8188, 7036, 5042, 8074, 118, 4308, 0] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[0, 8188, 7036, 5042, 8074, 118, 4308, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=218972160 dispatch_meta_bytes=107312 combine_payload_bytes=218972160 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=219079472 ingress_bytes=1628258304 link_bytes=1628258304 token_bytes=438051632 weight_movement_bytes=1409286144 total_nvlink_bytes=1847337776 weight_to_token_ratio=3.217169 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.117566834e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=163135488 dispatch_meta_bytes=79800 combine_payload_bytes=163135488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=163215288 ingress_bytes=1572421632 link_bytes=1572421632 token_bytes=326350776 weight_movement_bytes=1409286144 total_nvlink_bytes=1735636920 weight_to_token_ratio=4.318317 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.804904778e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32748 payload_rows_remote=27396 dup_rows=20 dup_rows_remote=0 dedup_ratio=0.000610 router_tokens_per_expert=[0, 1, 0, 5542, 11, 1, 2638, 8191] payload_rows_per_dst=[0, 2, 0, 6016, 22, 5070, 5352, 16286] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 20, 0] combine_rows_per_src=[0, 2, 0, 6016, 22, 5070, 5352, 16286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=202915840 dispatch_meta_bytes=99080 combine_payload_bytes=202915840 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=203014920 ingress_bytes=1612201984 link_bytes=1612201984 token_bytes=405930760 weight_movement_bytes=1409286144 total_nvlink_bytes=1815216904 weight_to_token_ratio=3.471740 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.027658144e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=16290 dup_rows=96 dup_rows_remote=0 dedup_ratio=0.002930 router_tokens_per_expert=[0, 1, 0, 6988, 13, 1, 1190, 8191] payload_rows_per_dst=[0, 2, 0, 7142, 26, 6546, 2574, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 96] combine_rows_per_src=[0, 2, 0, 7142, 26, 6546, 2574, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=227115008 dispatch_meta_bytes=110896 combine_payload_bytes=227115008 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=227225904 ingress_bytes=5159616512 link_bytes=5159616512 token_bytes=454340912 weight_movement_bytes=4932501504 total_nvlink_bytes=5386842416 weight_to_token_ratio=10.856389 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.889169874e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32738 payload_rows_remote=16384 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000916 router_tokens_per_expert=[1, 2134, 0, 0, 6072, 0, 1, 8176] payload_rows_per_dst=[16354, 16382, 0, 0, 0, 0, 2, 0] dup_rows_per_dst=[0, 30, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16354, 16382, 0, 0, 0, 0, 2, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 2, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 3364, 0, 0, 4837, 0, 0, 8183] payload_rows_per_dst=[0, 0, 50, 10654, 5954, 9382, 6728, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 50, 10654, 5954, 9382, 6728, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=133447680 dispatch_meta_bytes=65160 combine_payload_bytes=133447680 prefetch_bytes=704643072 reduce_bytes=5637144576 egress_bytes=133512840 ingress_bytes=6475235328 link_bytes=6475235328 token_bytes=266960520 weight_movement_bytes=6341787648 total_nvlink_bytes=6608748168 weight_to_token_ratio=23.755526 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.625861494e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=224428032 dispatch_meta_bytes=109584 combine_payload_bytes=224428032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=224537616 ingress_bytes=1633714176 link_bytes=1633714176 token_bytes=448965648 weight_movement_bytes=1409286144 total_nvlink_bytes=1858251792 weight_to_token_ratio=3.138962 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.148117440e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=16426 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[9, 3795, 0, 0, 4393, 0, 10, 8177] payload_rows_per_dst=[30, 7590, 16342, 5242, 3544, 0, 20, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[30, 7590, 16342, 5242, 3544, 0, 20, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134217728 dispatch_meta_bytes=65656 combine_payload_bytes=134217728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134283384 ingress_bytes=1543503872 link_bytes=1543503872 token_bytes=268501112 weight_movement_bytes=1409286144 total_nvlink_bytes=1677787256 weight_to_token_ratio=5.248716 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.642977393e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=18998 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 2913, 0, 1, 5310, 0, 0, 8160] payload_rows_per_dst=[0, 0, 0, 8156, 5016, 13770, 5826, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 0, 0, 8156, 5016, 13770, 5826, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=219660288 dispatch_meta_bytes=107256 combine_payload_bytes=219660288 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=219767544 ingress_bytes=4447518720 link_bytes=4447518720 token_bytes=439427832 weight_movement_bytes=4227858432 total_nvlink_bytes=4667286264 weight_to_token_ratio=9.621281 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.490424835e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=27456 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 3, 0, 5324, 27, 3, 2838, 8189] payload_rows_per_dst=[16378, 5312, 5342, 0, 54, 6, 5676, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 5312, 5342, 0, 54, 6, 5676, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28404 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 3737, 0, 0, 4459, 0, 0, 8188] payload_rows_per_dst=[0, 1454, 16376, 4364, 4554, 0, 6020, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[0, 1454, 16376, 4364, 4554, 0, 6020, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4946686168 prefetch_bytes=19730006016 reduce_bytes=11274289152 weight_movement_bytes=31004295168 egress_bytes=2473947352 ingress_bytes=33477033984 link_bytes=33477033984 total_nvlink_bytes=35950981336 weight_to_token_ratio=6.267690 prefetch_share_of_ingress=0.589359 dedup_ratio=0.001286 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=16 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019886 gemm_inner_pad_upper_bound=7366 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.171608852e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=134561792 dispatch_meta_bytes=65704 combine_payload_bytes=134561792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=134627496 ingress_bytes=1543847936 link_bytes=1543847936 token_bytes=269189288 weight_movement_bytes=1409286144 total_nvlink_bytes=1678475432 weight_to_token_ratio=5.235298 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.644904008e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32602 payload_rows_remote=16224 dup_rows=166 dup_rows_remote=0 dedup_ratio=0.005066 router_tokens_per_expert=[1, 4109, 0, 0, 4084, 0, 1, 8189] payload_rows_per_dst=[2, 0, 0, 0, 3796, 4206, 8220, 16378] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 166] combine_rows_per_src=[2, 0, 0, 0, 3796, 4206, 8220, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7257197504 prefetch_bytes=17616076800 reduce_bytes=21843935232 weight_movement_bytes=39460012032 egress_bytes=3629484992 ingress_bytes=43087724544 link_bytes=43087724544 total_nvlink_bytes=46717209536 weight_to_token_ratio=5.437362 prefetch_share_of_ingress=0.408842 dedup_ratio=0.000652 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=31 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020360 gemm_inner_pad_upper_bound=7620 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.507957948e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=26822 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1, 2972, 0, 0, 5232, 0, 1, 8178] payload_rows_per_dst=[2, 0, 0, 66, 5120, 5410, 5946, 16224] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[2, 0, 0, 66, 5120, 5410, 5946, 16224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=155631616 dispatch_meta_bytes=75992 combine_payload_bytes=155631616 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=155707608 ingress_bytes=1564917760 link_bytes=1564917760 token_bytes=311339224 weight_movement_bytes=1409286144 total_nvlink_bytes=1720625368 weight_to_token_ratio=4.526529 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.030303 gemm_active_groups=2 gemm_peak_group_rows=0 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.762886227e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232685568 dispatch_meta_bytes=113616 combine_payload_bytes=232685568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232799184 ingress_bytes=1641971712 link_bytes=1641971712 token_bytes=465484752 weight_movement_bytes=1409286144 total_nvlink_bytes=1874770896 weight_to_token_ratio=3.027567 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.022727 gemm_active_groups=3 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.194356194e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=224919552 dispatch_meta_bytes=109824 combine_payload_bytes=224919552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=225029376 ingress_bytes=1634205696 link_bytes=1634205696 token_bytes=449948928 weight_movement_bytes=1409286144 total_nvlink_bytes=1859235072 weight_to_token_ratio=3.132102 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.150869747e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6839118296 prefetch_bytes=21139292160 reduce_bytes=12683575296 weight_movement_bytes=33822867456 egress_bytes=3420400088 ingress_bytes=37241585664 link_bytes=37241585664 total_nvlink_bytes=40661985752 weight_to_token_ratio=4.945501 prefetch_share_of_ingress=0.567626 dedup_ratio=0.011356 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=18 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=7239 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.303358339e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=132907008 dispatch_meta_bytes=64896 combine_payload_bytes=132907008 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=132971904 ingress_bytes=5770051584 link_bytes=5770051584 token_bytes=265878912 weight_movement_bytes=5637144576 total_nvlink_bytes=5903023488 weight_to_token_ratio=21.201924 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.230988033e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=219725824 dispatch_meta_bytes=107288 combine_payload_bytes=219725824 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=219833112 ingress_bytes=1629011968 link_bytes=1629011968 token_bytes=439558936 weight_movement_bytes=1409286144 total_nvlink_bytes=1848845080 weight_to_token_ratio=3.206137 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.121787038e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6958446552 prefetch_bytes=14797504512 reduce_bytes=33118224384 weight_movement_bytes=47915728896 egress_bytes=3480074200 ingress_bytes=51394101248 link_bytes=51394101248 total_nvlink_bytes=54874175448 weight_to_token_ratio=6.885981 prefetch_share_of_ingress=0.287922 dedup_ratio=0.001644 prefetch_slots=42 unused_prefetch_slots=22 slot_utilization=0.656250 reduce_slots=47 remote_demand=21 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.023201 gemm_inner_pad_upper_bound=7366 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.798659463e-04 prefetch_bytes_per_gemm_flop=5.178740530e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7001379696 prefetch_bytes=18320719872 reduce_bytes=24662507520 weight_movement_bytes=42983227392 egress_bytes=3501544304 ingress_bytes=46483062784 link_bytes=46483062784 total_nvlink_bytes=49984607088 weight_to_token_ratio=6.139251 prefetch_share_of_ingress=0.394138 dedup_ratio=0.000038 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=35 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=7366 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.626785929e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=4772781696 prefetch_bytes=18320719872 reduce_bytes=28185722880 weight_movement_bytes=46506442752 egress_bytes=2386976384 ingress_bytes=48892248064 link_bytes=48892248064 total_nvlink_bytes=51279224448 weight_to_token_ratio=9.744096 prefetch_share_of_ingress=0.374716 dedup_ratio=0.007118 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=40 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019886 gemm_inner_pad_upper_bound=7620 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.711101128e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6481488896 prefetch_bytes=21139292160 reduce_bytes=5637144576 weight_movement_bytes=26776436736 egress_bytes=3241536512 ingress_bytes=30016389120 link_bytes=30016389120 total_nvlink_bytes=33257925632 weight_to_token_ratio=4.131217 prefetch_share_of_ingress=0.704258 dedup_ratio=0.001522 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=8 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020360 gemm_inner_pad_upper_bound=6985 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.050495310e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=25456 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[3, 2387, 0, 0, 5804, 0, 3, 8187] payload_rows_per_dst=[16380, 7312, 0, 4286, 4784, 0, 6, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16380, 7312, 0, 4286, 4784, 0, 6, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=208535552 dispatch_meta_bytes=101824 combine_payload_bytes=208535552 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=208637376 ingress_bytes=3027107840 link_bytes=3027107840 token_bytes=417172928 weight_movement_bytes=2818572288 total_nvlink_bytes=3235745216 weight_to_token_ratio=6.756364 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=4736 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.695054033e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=6911868840 prefetch_bytes=20434649088 reduce_bytes=14092861440 weight_movement_bytes=34527510528 egress_bytes=3456778152 ingress_bytes=37982601216 link_bytes=37982601216 total_nvlink_bytes=41439379368 weight_to_token_ratio=4.995394 prefetch_share_of_ingress=0.538000 dedup_ratio=0.000198 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=20 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=6985 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.329291951e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none - [2026-09-16 21:34:45] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 596.8 | throughput per GPU (TFLOP/s/GPU): 558.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.287129E+00 | loss scale: 1.0 | grad norm: 5.199 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-09-16 21:34:45 -[rank0]:[W916 21:34:47.473824171 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W916 21:34:48.636253871 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W916 21:34:48.639768755 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W916 21:34:48.642336018 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W916 21:34:48.643714180 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W916 21:34:48.682661755 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W916 21:34:48.066849622 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W916 21:34:48.067481164 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0916 21:34:52.502000 28497 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0916 21:34:52.502000 28497 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 28685 closing signal SIGTERM -W0916 21:34:52.503000 28497 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 28686 closing signal SIGTERM -W0916 21:34:52.503000 28497 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 28687 closing signal SIGTERM -W0916 21:34:52.503000 28497 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 28688 closing signal SIGTERM -W0916 21:34:52.503000 28497 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 28689 closing signal SIGTERM -W0916 21:34:52.504000 28497 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 28690 closing signal SIGTERM -W0916 21:34:52.504000 28497 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 28691 closing signal SIGTERM -W0916 21:34:52.505000 28497 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 28692 closing signal SIGTERM -Traceback (most recent call last): - File "/usr/local/bin/torchrun", line 8, in - sys.exit(main()) - ^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper - return f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main - run(args) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run - elastic_launch( - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent - result = agent.run() - ^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper - result = f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run - result = self._invoke_run(role) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run - time.sleep(monitor_interval) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 28497 got signal: 15 +[21:15:37] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[21:15:37] WARN: MoonEP source is on 'megatron-integration', need 'megatron-ddp-symmetric-params' +[21:15:37] ERROR: MoonEP source must be /workspace/MoonEP @ megatron-ddp-symmetric-params diff --git a/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1/train.log b/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1/train.log index 2a21059be893bd5c86129e05709f491390a03a07..2e5d858c578865d056fcc00b02b6044415a356ce 100644 --- a/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1/train.log +++ b/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__forcebal__async/rep1/train.log @@ -11,4823 +11,3 @@ W0916 21:35:04.783000 31788 torch/distributed/run.py:766] W0916 21:35:04.783000 31788 torch/distributed/run.py:766] ***************************************** W0916 21:35:04.783000 31788 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0916 21:35:04.783000 31788 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.auto - attention_dropout ............................... 0.0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_scope ................................ full - cuda_graph_warmup_steps ......................... 3 - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 8 - data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_type ................................. single - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - delay_wgrad_compute ............................. False - deprecated_use_mcore_models ..................... True - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_bf16_reduced_precision_matmul ........... False - disable_mamba_mem_eff_path ...................... False - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - embedding_init_method_std ....................... None - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_experimental ............................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_msc ...................................... True - enable_one_logger ............................... True - encoder_num_layers .............................. 16 - encoder_seq_length .............................. 4096 - end_weight_decay ................................ 0.1 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000000 - eval_iters ...................................... 0 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 8 - expert_tensor_parallel_size ..................... 1 - export_force_local_attention .................... False - export_kd_cfg ................................... None - export_kd_teacher_ckpt_format ................... None - export_kd_teacher_load .......................... None - export_kv_cache_quant ........................... False - export_legacy_megatron .......................... False - export_model_type ............................... GPTModel - export_moe_apply_probs_on_input ................. False - export_qk_l2_norm ............................... False - export_quant_cfg ................................ None - export_real_quant_cfg ........................... None - export_te_mcore_model ........................... False - external_cuda_graph ............................. False - ffn_hidden_size ................................. 14336 - finetune ........................................ False - finetune_data_split ............................. train - finetune_hf_dataset ............................. None - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. e4m3 - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... mxfp8 - fp8_wgrad ....................................... True - fsdp_double_buffer .............................. False - global_batch_size ............................... 16 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... True - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - heterogeneous_layers_config_encoded_json ........ None - heterogeneous_layers_config_path ................ None - hidden_dropout .................................. 0.0 - hidden_size ..................................... 4096 - hierarchical_context_parallel_sizes ............. None - high_priority_stream_groups ..................... [] - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_dynamic_batching ...................... False - inference_dynamic_batching_buffer_guaranteed_fraction 0.2 - inference_dynamic_batching_buffer_overflow_factor None - inference_dynamic_batching_buffer_size_gb ....... 40.0 - inference_dynamic_batching_chunk_size ........... 256 - inference_dynamic_batching_max_requests_override None - inference_dynamic_batching_max_tokens_override .. None - inference_dynamic_batching_num_cuda_graphs ...... 16 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.01 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 4294967296 - inprocess_active_world_size ..................... 8 - inprocess_barrier_timeout ....................... 120 - inprocess_completion_timeout .................... 120 - inprocess_empty_cuda_cache ...................... False - inprocess_granularity ........................... node - inprocess_hard_timeout .......................... 90 - inprocess_heartbeat_interval .................... 30 - inprocess_heartbeat_timeout ..................... 60 - inprocess_last_call_wait ........................ 1 - inprocess_max_iterations ........................ None - inprocess_monitor_process_interval .............. 1.0 - inprocess_monitor_thread_interval ............... 1.0 - inprocess_progress_watchdog_interval ............ 1.0 - inprocess_restart ............................... False - inprocess_soft_timeout .......................... 60 - inprocess_termination_grace_time ................ 1 - is_hybrid_model ................................. False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kitchen_config_file ............................. None - kitchen_recipe_number ........................... None - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - lazy_mpu_init ................................... None - load ............................................ None - load_main_params_from_ckpt ...................... None - load_model_opt_format ........................... False - local_rank ...................................... 0 - log_energy ...................................... False - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. True - log_timers_to_tensorboard ....................... False - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 8 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. None - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 5 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - mamba_head_dim .................................. 64 - mamba_num_groups ................................ 8 - mamba_num_heads ................................. None - mamba_state_dim ................................. 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... False - max_position_embeddings ......................... 32768 - max_tokens_to_oom ............................... 12000 - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - mid_level_dataset_surplus ....................... 0.005 - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-05 - mlp_chunks_for_prefill .......................... 1 - mmap_bin_files .................................. True - mock_data ....................................... False - moe_apply_probs_on_input ........................ False - moe_aux_loss_coeff .............................. 0.01 - moe_deepep_num_sms .............................. 20 - moe_echo_enable_random_offloading ............... False - moe_echo_expert_dispatch_overlap ................ False - moe_echo_expert_dispatcher_type ................. hybridep - moe_echo_recompute_expert_dispatch .............. False - moe_enable_deepep ............................... False - moe_enable_echo ................................. False - moe_enable_hybridep ............................. False - moe_enable_moonep ............................... True - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. 14336 - moe_grouped_gemm ................................ True - moe_hybridep_num_sms ............................ 16 - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_moonep_async_grad_reduce .................... True - moe_moonep_num_sms .............................. None - moe_moonep_traffic_logging ...................... True - moe_moonep_traffic_logging_interval ............. 1 - moe_moonep_traffic_warmup_iters ................. 2 - moe_num_echo_experts ............................ None - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_received_token_capacity ..................... None - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ fp32 - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. True - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... flex - moe_token_drop_policy ........................... probs - moe_upcycling_granularity ....................... 1 - moe_use_device_initiated_grouped_gemm ........... True - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mrope_section ................................... None - mscale .......................................... 1.0 - mscale_all_dim .................................. 0.0 - mtp_loss_scaling_factor ......................... 0.1 - mtp_num_layers .................................. None - multi_latent_attention .......................... False - nccl_all_reduce_for_prefill ..................... False - nccl_communicator_config_path ................... None - nccl_ub ......................................... False - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_rope_freq .................................... None - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-05 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... 8 - num_layers ...................................... 16 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 2 - object_storage_cache_path ....................... None - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_grad_reduce ............................. True - overlap_moe_expert_parallel_comm ................ False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ True - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_layout .................. None - pipeline_model_parallel_size .................... 1 - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_l2_norm ...................................... False - qk_layernorm .................................... False - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... None - recompute_method ................................ None - recompute_modules ............................... None - recompute_num_layers ............................ None - record_memory_history ........................... False - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... validate_results - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - reuse_grad_buf_for_mxfp8_param_ag ............... False - rope_scaling_factor ............................. 8.0 - rope_type ....................................... None - rotary_base ..................................... 1000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - run_workload_inspector_server ................... False - sample_rate ..................................... 1.0 - save ............................................ None - save_interval ................................... None - save_retain_interval ............................ None - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - seq_length ...................................... 4096 - sequence_parallel ............................... False - sft ............................................. False - sft_tokenizer_prompt_format ..................... nemotron-h-aligned - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - squared_relu .................................... False - start_weight_decay .............................. 0.1 - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - suggested_communication_unit_size ............... None - swiglu .......................................... True - swin_backbone_type .............................. tiny - symmetric_ar_type ............................... None - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. None - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. GPT2BPETokenizer - torch_fsdp2_reshard_after_forward ............... True - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 12 - train_samples ................................... None - train_sync_interval ............................. None - transformer_impl ................................ transformer_engine - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... True - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_sharp ....................................... False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ False - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json - vocab_size ...................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.1 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 8 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> building GPT2BPETokenizer tokenizer ... - > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS -> initializing torch distributed ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 74 - is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank4]:[W916 21:35:20.033629549 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W916 21:35:20.033827802 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W916 21:35:20.033861294 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W916 21:35:20.033889808 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W916 21:35:20.033973553 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W916 21:35:20.033980156 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W916 21:35:20.034429922 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -> compiling dataset index builder ... -make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.035 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -[rank0]:[W916 21:35:21.198968481 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.746 seconds -time to initialize megatron (seconds): 7.459 -[after megatron is initialized] datetime: 2026-09-16 21:35:27 -building GPT model ... -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3902410752 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 -Params for bucket 1 (206045184 elements, 206045184 padded size): - module.output_layer.weight -Params for bucket 2 (41988096 elements, 41988096 padded size): - module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.final_layernorm.weight - module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.self_attention.linear_qkv.weight - module.decoder.layers.15.mlp.router.weight - module.decoder.layers.15.self_attention.linear_proj.weight -Params for bucket 3 (41984000 elements, 41984000 padded size): - module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.14.self_attention.linear_qkv.weight - module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight -Params for bucket 4 (41984000 elements, 41984000 padded size): - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.13.self_attention.linear_qkv.weight - module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.13.pre_mlp_layernorm.weight -Params for bucket 5 (41984000 elements, 41984000 padded size): - module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.12.self_attention.linear_qkv.weight - module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight -Params for bucket 6 (41984000 elements, 41984000 padded size): - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.self_attention.linear_qkv.weight - module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.11.pre_mlp_layernorm.weight -Params for bucket 7 (41984000 elements, 41984000 padded size): - module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight -Params for bucket 8 (41984000 elements, 41984000 padded size): - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.self_attention.linear_qkv.weight - module.decoder.layers.9.pre_mlp_layernorm.weight -Params for bucket 9 (41984000 elements, 41984000 padded size): - module.decoder.layers.8.pre_mlp_layernorm.weight - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight -Params for bucket 10 (41984000 elements, 41984000 padded size): - module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_qkv.weight -Params for bucket 11 (41984000 elements, 41984000 padded size): - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.linear_proj.weight -Params for bucket 12 (41984000 elements, 41984000 padded size): - module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.self_attention.linear_qkv.weight -Params for bucket 13 (41984000 elements, 41984000 padded size): - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight -Params for bucket 14 (41984000 elements, 41984000 padded size): - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.3.pre_mlp_layernorm.weight -Params for bucket 15 (41984000 elements, 41984000 padded size): - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight -Params for bucket 16 (41984000 elements, 41984000 padded size): - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.linear_qkv.weight -Params for bucket 17 (41984000 elements, 41984000 padded size): - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_proj.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.self_attention.linear_qkv.weight -Params for bucket 18 (206045184 elements, 206045184 padded size): - module.embedding.word_embeddings.weight -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 32 -Params for bucket 1 (58720256 elements, 58720256 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight0 -Params for bucket 2 (117440512 elements, 117440512 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 -Params for bucket 3 (58720256 elements, 58720256 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight0 -Params for bucket 4 (117440512 elements, 117440512 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 -Params for bucket 5 (58720256 elements, 58720256 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 -Params for bucket 6 (117440512 elements, 117440512 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight0 -Params for bucket 7 (58720256 elements, 58720256 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight0 -Params for bucket 8 (117440512 elements, 117440512 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight0 -Params for bucket 9 (58720256 elements, 58720256 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 -Params for bucket 10 (117440512 elements, 117440512 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 -Params for bucket 11 (58720256 elements, 58720256 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 -Params for bucket 12 (117440512 elements, 117440512 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 -Params for bucket 13 (58720256 elements, 58720256 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 -Params for bucket 14 (117440512 elements, 117440512 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 -Params for bucket 15 (58720256 elements, 58720256 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 -Params for bucket 16 (117440512 elements, 117440512 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 -Params for bucket 17 (58720256 elements, 58720256 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 -Params for bucket 18 (117440512 elements, 117440512 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 -Params for bucket 19 (58720256 elements, 58720256 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 -Params for bucket 20 (117440512 elements, 117440512 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 -Params for bucket 21 (58720256 elements, 58720256 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight0 -Params for bucket 22 (117440512 elements, 117440512 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 -Params for bucket 23 (58720256 elements, 58720256 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 -Params for bucket 24 (117440512 elements, 117440512 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 -Params for bucket 25 (58720256 elements, 58720256 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 -Params for bucket 26 (117440512 elements, 117440512 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 -Params for bucket 27 (58720256 elements, 58720256 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 -Params for bucket 28 (117440512 elements, 117440512 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 -Params for bucket 29 (58720256 elements, 58720256 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 -Params for bucket 30 (117440512 elements, 117440512 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 -Params for bucket 31 (58720256 elements, 58720256 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 -Params for bucket 32 (117440512 elements, 117440512 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-16 21:35:29 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 192 - validation: 0 - test: 0 -INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] -> building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) -INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers -INFO:megatron.core.datasets.indexed_dataset: Extract the document indices -INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 -INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 -INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices -INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy -INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy -INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy -INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 -> finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-09-16 21:35:30 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (2863.70, 2867.68) - train/valid/test-data-iterators-setup ..........: (69.59, 85.43) -training ... -Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-09-16 21:35:30 -INFO:moonep.api:MoonEP comm buffer sizes: - config : R=8, S=4096, H=4096, E=8, K=2, NvS=8446, NvS_padded=8448 - sm_count : num_sms=32, num_sms_dedup=148 - hidden_buffer_size : 66.00 MiB - meta_buffer_size : 2.00 MiB - local_temp_buffer_size : 512.66 KiB - total_size : 68.50 MiB -[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 8] sizes [8192, 128] | duplicated experts [[5], [0], [0], [-1], [3], [-1], [2], [0]] -[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 8] sizes [8192, 128] | duplicated experts [[2], [-1], [1], [7], [2], [1], [1], [5]] -[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 8] sizes [8192, 128] | duplicated experts [[6], [5], [6], [5], [2], [6], [-1], [4]] -[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 8] sizes [8192, 128] | duplicated experts [[3], [4], [4], [4], [-1], [7], [2], [0]] -WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-09-16 21:35:50] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 20245.6 | throughput per GPU (TFLOP/s/GPU): 16.5 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.104439E+01 | loss scale: 1.0 | grad norm: 33.889 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 23.22 -Number of parameters in embedding layers in billions: 0.41 -Total number of parameters in billions: 23.63 -Number of parameters in most loaded shard in billions: 23.6320 -Theoretical memory footprints: weight and optimizer=169029.43 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 53315.5478515625 | max allocated: 53315.548828125 | reserved: 59020.0 | max reserved: 59020.0 - [2026-09-16 21:35:50] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 592.1 | throughput per GPU (TFLOP/s/GPU): 563.0 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.105176E+01 | loss scale: 1.0 | grad norm: 36.174 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=28456 dup_rows=98 dup_rows_remote=78 dedup_ratio=0.002991 router_tokens_per_expert=[2054, 2073, 2043, 1995, 2082, 2013, 2060, 2064] payload_rows_per_dst=[4214, 4222, 4128, 3816, 4356, 4062, 3982, 3890] dup_rows_per_dst=[20, 18, 4, 0, 42, 14, 0, 0] combine_rows_per_src=[4214, 4222, 4128, 3816, 4356, 4062, 3982, 3890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28478 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2045, 2026, 2145, 2064, 1962, 2048, 2073] payload_rows_per_dst=[4042, 4090, 4052, 4290, 4128, 3924, 4096, 4146] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4090, 4052, 4290, 4128, 3924, 4096, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2105, 2117, 1992, 1955, 2039, 2035, 2086] payload_rows_per_dst=[4110, 4210, 4234, 3984, 3910, 4078, 4070, 4172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4210, 4234, 3984, 3910, 4078, 4070, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2000, 2010, 2061, 2111, 2090, 2077, 2001, 2034] payload_rows_per_dst=[4000, 4020, 4122, 4222, 4180, 4154, 4002, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4000, 4020, 4122, 4222, 4180, 4154, 4002, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32714 payload_rows_remote=28426 dup_rows=54 dup_rows_remote=22 dedup_ratio=0.001648 router_tokens_per_expert=[2070, 2055, 2023, 2082, 2045, 2075, 2038, 1996] payload_rows_per_dst=[4130, 4110, 3694, 4188, 4020, 4150, 4134, 4288] dup_rows_per_dst=[0, 0, 0, 10, 0, 0, 12, 32] combine_rows_per_src=[4130, 4110, 3694, 4188, 4020, 4150, 4134, 4288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28842 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 1963, 2118, 2018, 2068, 2069, 2036, 2046] payload_rows_per_dst=[4132, 3926, 4236, 4036, 4136, 4138, 4072, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 3926, 4236, 4036, 4136, 4138, 4072, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28598 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2102, 2051, 2085, 2052, 1979, 2011, 2089, 2015] payload_rows_per_dst=[4204, 4102, 4170, 4104, 3958, 4022, 4178, 4030] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4204, 4102, 4170, 4104, 3958, 4022, 4178, 4030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233111552 dispatch_meta_bytes=114136 combine_payload_bytes=233111552 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233225688 ingress_bytes=2347040768 link_bytes=2347040768 token_bytes=466337240 weight_movement_bytes=2113929216 total_nvlink_bytes=2580266456 weight_to_token_ratio=4.533048 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314244860e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233291776 dispatch_meta_bytes=113912 combine_payload_bytes=233291776 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233405688 ingress_bytes=3051864064 link_bytes=3051864064 token_bytes=466697464 weight_movement_bytes=2818572288 total_nvlink_bytes=3285269752 weight_to_token_ratio=6.039399 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708916485e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235208808 ingress_bytes=3053666304 link_bytes=3053666304 token_bytes=470302824 weight_movement_bytes=2818572288 total_nvlink_bytes=3288875112 weight_to_token_ratio=5.993101 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709925664e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1958, 2035, 2064, 2059, 2019, 2113, 2111, 2025] payload_rows_per_dst=[3916, 4070, 4128, 4118, 4038, 4226, 4222, 4050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3916, 4070, 4128, 4118, 4038, 4226, 4222, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234520344 ingress_bytes=2348335104 link_bytes=2348335104 token_bytes=468926232 weight_movement_bytes=2113929216 total_nvlink_bytes=2582855448 weight_to_token_ratio=4.508021 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314969634e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=232865792 dispatch_meta_bytes=113792 combine_payload_bytes=232865792 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232979584 ingress_bytes=2346795008 link_bytes=2346795008 token_bytes=465845376 weight_movement_bytes=2113929216 total_nvlink_bytes=2579774592 weight_to_token_ratio=4.537834 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314107244e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=236273664 dispatch_meta_bytes=115368 combine_payload_bytes=236273664 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236389032 ingress_bytes=1645559808 link_bytes=1645559808 token_bytes=472662696 weight_movement_bytes=1409286144 total_nvlink_bytes=1881948840 weight_to_token_ratio=2.981590 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.214448034e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114392 combine_payload_bytes=234274816 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234389208 ingress_bytes=3757490176 link_bytes=3757490176 token_bytes=468664024 weight_movement_bytes=3523215360 total_nvlink_bytes=3991879384 weight_to_token_ratio=7.517572 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104037653e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32652 payload_rows_remote=28440 dup_rows=116 dup_rows_remote=78 dedup_ratio=0.003540 router_tokens_per_expert=[2003, 2053, 2086, 2121, 2089, 1992, 2044, 1996] payload_rows_per_dst=[4212, 4118, 4306, 4092, 3918, 4280, 4080, 3646] dup_rows_per_dst=[38, 2, 20, 2, 0, 54, 0, 0] combine_rows_per_src=[4212, 4118, 4306, 4092, 3918, 4280, 4080, 3646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=114920 combine_payload_bytes=235356160 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235471080 ingress_bytes=3053928448 link_bytes=3053928448 token_bytes=470827240 weight_movement_bytes=2818572288 total_nvlink_bytes=3289399528 weight_to_token_ratio=5.986426 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710072454e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2035, 2069, 2057, 2076, 2016, 2012, 2062] payload_rows_per_dst=[4114, 4070, 4138, 4114, 4152, 4032, 4024, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4070, 4138, 4114, 4152, 4032, 4024, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2018, 2026, 1996, 2086, 2049, 2065, 2107] payload_rows_per_dst=[4074, 4036, 4052, 3992, 4172, 4098, 4130, 4214] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4036, 4052, 3992, 4172, 4098, 4130, 4214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=232980480 dispatch_meta_bytes=114072 combine_payload_bytes=232980480 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233094552 ingress_bytes=1642266624 link_bytes=1642266624 token_bytes=466075032 weight_movement_bytes=1409286144 total_nvlink_bytes=1875361176 weight_to_token_ratio=3.023732 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196007579e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=28578 dup_rows=24 dup_rows_remote=4 dedup_ratio=0.000732 router_tokens_per_expert=[2087, 2094, 2019, 2103, 2037, 2014, 2042, 1988] payload_rows_per_dst=[4174, 4068, 3902, 4206, 4126, 4028, 4074, 4166] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 20] combine_rows_per_src=[4174, 4068, 3902, 4206, 4126, 4028, 4074, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2080, 2022, 2014, 2027, 2046, 2047, 2091] payload_rows_per_dst=[4114, 4160, 4044, 4028, 4054, 4092, 4094, 4182] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4160, 4044, 4028, 4054, 4092, 4094, 4182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234848184 ingress_bytes=3053305856 link_bytes=3053305856 token_bytes=469581752 weight_movement_bytes=2818572288 total_nvlink_bytes=3288154040 weight_to_token_ratio=6.002304 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709723828e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28510 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1996, 2129, 2029, 2019, 2125, 2031, 2014, 2041] payload_rows_per_dst=[3992, 4258, 4058, 4038, 4250, 4062, 4028, 4082] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3992, 4258, 4058, 4038, 4250, 4062, 4028, 4082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28726 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 2007, 2021, 2100, 2055, 2038, 2063, 2069] payload_rows_per_dst=[4062, 4014, 4042, 4200, 4110, 4076, 4126, 4138] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 4014, 4042, 4200, 4110, 4076, 4126, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234717048 ingress_bytes=3053174784 link_bytes=3053174784 token_bytes=469319544 weight_movement_bytes=2818572288 total_nvlink_bytes=3287891832 weight_to_token_ratio=6.005657 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709650433e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114328 combine_payload_bytes=234110976 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234225304 ingress_bytes=2348040192 link_bytes=2348040192 token_bytes=468336280 weight_movement_bytes=2113929216 total_nvlink_bytes=2582265496 weight_to_token_ratio=4.513699 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314804495e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234913792 dispatch_meta_bytes=114704 combine_payload_bytes=234913792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235028496 ingress_bytes=1644199936 link_bytes=1644199936 token_bytes=469942288 weight_movement_bytes=1409286144 total_nvlink_bytes=1879228432 weight_to_token_ratio=2.998849 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206833319e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32724 payload_rows_remote=28512 dup_rows=44 dup_rows_remote=42 dedup_ratio=0.001343 router_tokens_per_expert=[2099, 2078, 1989, 2053, 2103, 1983, 2054, 2025] payload_rows_per_dst=[4212, 4206, 4058, 4114, 4110, 3966, 4094, 3964] dup_rows_per_dst=[2, 24, 16, 2, 0, 0, 0, 0] combine_rows_per_src=[4212, 4206, 4058, 4114, 4110, 3966, 4094, 3964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28786 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1994, 2021, 2103, 2048, 1991, 2023, 2106, 2098] payload_rows_per_dst=[3988, 4042, 4206, 4096, 3982, 4046, 4212, 4196] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3988, 4042, 4206, 4096, 3982, 4046, 4212, 4196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233553920 dispatch_meta_bytes=114040 combine_payload_bytes=233553920 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233667960 ingress_bytes=3756769280 link_bytes=3756769280 token_bytes=467221880 weight_movement_bytes=3523215360 total_nvlink_bytes=3990437240 weight_to_token_ratio=7.540776 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103633982e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235323392 dispatch_meta_bytes=114904 combine_payload_bytes=235323392 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235438296 ingress_bytes=2349252608 link_bytes=2349252608 token_bytes=470761688 weight_movement_bytes=2113929216 total_nvlink_bytes=2584690904 weight_to_token_ratio=4.490444 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315483398e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28636 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2067, 2037, 2053, 2066, 2054, 2061, 2067, 1979] payload_rows_per_dst=[4134, 4074, 4106, 4132, 4108, 4122, 4134, 3958] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4134, 4074, 4106, 4132, 4108, 4122, 4134, 3958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233570304 dispatch_meta_bytes=114216 combine_payload_bytes=233570304 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233684520 ingress_bytes=2347499520 link_bytes=2347499520 token_bytes=467254824 weight_movement_bytes=2113929216 total_nvlink_bytes=2581184040 weight_to_token_ratio=4.524146 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314501742e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235814912 dispatch_meta_bytes=115144 combine_payload_bytes=235814912 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235930056 ingress_bytes=3759030272 link_bytes=3759030272 token_bytes=471744968 weight_movement_bytes=3523215360 total_nvlink_bytes=3994960328 weight_to_token_ratio=7.468475 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104900043e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32690 payload_rows_remote=28416 dup_rows=78 dup_rows_remote=64 dedup_ratio=0.002380 router_tokens_per_expert=[2049, 2029, 2037, 2013, 2044, 2068, 2043, 2101] payload_rows_per_dst=[4026, 3836, 4236, 3856, 4094, 4256, 4112, 4274] dup_rows_per_dst=[0, 0, 40, 0, 12, 12, 0, 14] combine_rows_per_src=[4026, 3836, 4236, 3856, 4094, 4256, 4112, 4274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1969, 2057, 2032, 2036, 2060, 2070, 2073, 2087] payload_rows_per_dst=[3938, 4114, 4064, 4072, 4120, 4140, 4146, 4174] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3938, 4114, 4064, 4072, 4120, 4140, 4146, 4174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234586112 dispatch_meta_bytes=114544 combine_payload_bytes=234586112 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234700656 ingress_bytes=3053158400 link_bytes=3053158400 token_bytes=469286768 weight_movement_bytes=2818572288 total_nvlink_bytes=3287859056 weight_to_token_ratio=6.006077 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709641259e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2092, 2065, 2089, 2032, 2008, 2053, 1991] payload_rows_per_dst=[4108, 4184, 4130, 4178, 4064, 4016, 4106, 3982] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4184, 4130, 4178, 4064, 4016, 4106, 3982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2025, 2092, 2026, 2056, 2019, 1977, 2065, 2124] payload_rows_per_dst=[4050, 4184, 4052, 4112, 4038, 3954, 4130, 4248] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4050, 4184, 4052, 4112, 4038, 3954, 4130, 4248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 1990, 2014, 2086, 2018, 2113, 2028, 2077] payload_rows_per_dst=[4116, 3980, 4028, 4172, 4036, 4226, 4056, 4154] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 3980, 4028, 4172, 4036, 4226, 4056, 4154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=232783872 dispatch_meta_bytes=113920 combine_payload_bytes=232783872 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=232897792 ingress_bytes=3051356160 link_bytes=3051356160 token_bytes=465681664 weight_movement_bytes=2818572288 total_nvlink_bytes=3284253952 weight_to_token_ratio=6.052573 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708632080e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114488 combine_payload_bytes=234471424 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234585912 ingress_bytes=2348400640 link_bytes=2348400640 token_bytes=469057336 weight_movement_bytes=2113929216 total_nvlink_bytes=2582986552 weight_to_token_ratio=4.506761 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315006331e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=28520 dup_rows=98 dup_rows_remote=96 dedup_ratio=0.002991 router_tokens_per_expert=[2070, 2029, 2059, 1956, 2105, 2043, 2025, 2097] payload_rows_per_dst=[4150, 4260, 4026, 4244, 3966, 4086, 3924, 4014] dup_rows_per_dst=[2, 38, 0, 58, 0, 0, 0, 0] combine_rows_per_src=[4150, 4260, 4026, 4244, 3966, 4086, 3924, 4014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235651392 ingress_bytes=1644822528 link_bytes=1644822528 token_bytes=471187776 weight_movement_bytes=1409286144 total_nvlink_bytes=1880473920 weight_to_token_ratio=2.990923 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210319574e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2083, 2019, 2065, 2093, 2038, 1995, 2000] payload_rows_per_dst=[4182, 4166, 4038, 4130, 4186, 4076, 3990, 4000] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4166, 4038, 4130, 4186, 4076, 3990, 4000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234160128 dispatch_meta_bytes=114336 combine_payload_bytes=234160128 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234274464 ingress_bytes=3052732416 link_bytes=3052732416 token_bytes=468434592 weight_movement_bytes=2818572288 total_nvlink_bytes=3287006880 weight_to_token_ratio=6.017003 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709402726e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114960 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235553040 ingress_bytes=1644724224 link_bytes=1644724224 token_bytes=470991120 weight_movement_bytes=1409286144 total_nvlink_bytes=1880277264 weight_to_token_ratio=2.992171 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209769113e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28860 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2009, 2044, 1954, 2067, 2109, 2074, 2036] payload_rows_per_dst=[4182, 4018, 4088, 3908, 4134, 4218, 4148, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4018, 4088, 3908, 4134, 4218, 4148, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233635840 dispatch_meta_bytes=114464 combine_payload_bytes=233635840 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233750304 ingress_bytes=3756851200 link_bytes=3756851200 token_bytes=467386144 weight_movement_bytes=3523215360 total_nvlink_bytes=3990601504 weight_to_token_ratio=7.538125 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103679853e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234143744 dispatch_meta_bytes=114328 combine_payload_bytes=234143744 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234258072 ingress_bytes=3052716032 link_bytes=3052716032 token_bytes=468401816 weight_movement_bytes=2818572288 total_nvlink_bytes=3286974104 weight_to_token_ratio=6.017424 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709393551e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32652 payload_rows_remote=28474 dup_rows=116 dup_rows_remote=96 dedup_ratio=0.003540 router_tokens_per_expert=[2093, 2072, 2070, 2040, 2077, 2002, 2005, 2025] payload_rows_per_dst=[3754, 4144, 4110, 4080, 4018, 4342, 4026, 4178] dup_rows_per_dst=[0, 0, 0, 0, 0, 84, 12, 20] combine_rows_per_src=[3754, 4144, 4110, 4080, 4018, 4342, 4026, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2088, 2041, 2120, 2024, 2116, 1928, 2004, 2063] payload_rows_per_dst=[4176, 4082, 4240, 4048, 4232, 3856, 4008, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4176, 4082, 4240, 4048, 4232, 3856, 4008, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=236421120 dispatch_meta_bytes=115440 combine_payload_bytes=236421120 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236536560 ingress_bytes=1645707264 link_bytes=1645707264 token_bytes=472957680 weight_movement_bytes=1409286144 total_nvlink_bytes=1882243824 weight_to_token_ratio=2.979730 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.215273726e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28628 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2067, 2061, 2035, 2028, 2070, 2057, 1995] payload_rows_per_dst=[4142, 4134, 4122, 4070, 4056, 4140, 4114, 3990] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4134, 4122, 4070, 4056, 4140, 4114, 3990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233259008 dispatch_meta_bytes=114280 combine_payload_bytes=233259008 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233373288 ingress_bytes=2347188224 link_bytes=2347188224 token_bytes=466632296 weight_movement_bytes=2113929216 total_nvlink_bytes=2580561512 weight_to_token_ratio=4.530182 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314327429e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2015, 2026, 1970, 2078, 2081, 2013, 2137, 2064] payload_rows_per_dst=[4030, 4052, 3940, 4156, 4162, 4026, 4274, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4030, 4052, 3940, 4156, 4162, 4026, 4274, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235601920 dispatch_meta_bytes=115040 combine_payload_bytes=235601920 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235716960 ingress_bytes=2349531136 link_bytes=2349531136 token_bytes=471318880 weight_movement_bytes=2113929216 total_nvlink_bytes=2585248096 weight_to_token_ratio=4.485136 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315639362e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=28606 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000732 router_tokens_per_expert=[2064, 2050, 2059, 2019, 2039, 2028, 2081, 2044] payload_rows_per_dst=[4138, 4100, 4130, 4206, 4056, 4056, 4162, 3896] dup_rows_per_dst=[0, 0, 0, 24, 0, 0, 0, 0] combine_rows_per_src=[4138, 4100, 4130, 4206, 4056, 4056, 4162, 3896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28724 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2110, 1985, 2022, 2049, 2051, 2038, 2070, 2059] payload_rows_per_dst=[4220, 3970, 4044, 4098, 4102, 4076, 4140, 4118] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4220, 3970, 4044, 4098, 4102, 4076, 4140, 4118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2035, 2099, 2053, 2049, 1970, 2061, 2061] payload_rows_per_dst=[4112, 4070, 4198, 4106, 4098, 3940, 4122, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4070, 4198, 4106, 4098, 3940, 4122, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234520576 dispatch_meta_bytes=114512 combine_payload_bytes=234520576 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234635088 ingress_bytes=1643806720 link_bytes=1643806720 token_bytes=469155664 weight_movement_bytes=1409286144 total_nvlink_bytes=1878441808 weight_to_token_ratio=3.003878 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204631473e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28798 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2013, 2061, 2027, 1985, 2103, 2076, 2110, 2009] payload_rows_per_dst=[4026, 4122, 4054, 3970, 4206, 4152, 4220, 4018] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4026, 4122, 4054, 3970, 4206, 4152, 4220, 4018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235356336 ingress_bytes=1644527616 link_bytes=1644527616 token_bytes=470597808 weight_movement_bytes=1409286144 total_nvlink_bytes=1879883952 weight_to_token_ratio=2.994672 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208668190e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114520 combine_payload_bytes=234340352 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234454872 ingress_bytes=4462198784 link_bytes=4462198784 token_bytes=468795224 weight_movement_bytes=4227858432 total_nvlink_bytes=4696653656 weight_to_token_ratio=9.018561 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498645058e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235307008 dispatch_meta_bytes=114896 combine_payload_bytes=235307008 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235421904 ingress_bytes=3758522368 link_bytes=3758522368 token_bytes=470728912 weight_movement_bytes=3523215360 total_nvlink_bytes=3993944272 weight_to_token_ratio=7.484595 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104615638e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28414 dup_rows=80 dup_rows_remote=66 dedup_ratio=0.002441 router_tokens_per_expert=[2138, 2035, 1979, 1940, 2087, 2064, 2056, 2085] payload_rows_per_dst=[4048, 4234, 3828, 3904, 4004, 4310, 4086, 4274] dup_rows_per_dst=[0, 22, 0, 8, 0, 30, 6, 14] combine_rows_per_src=[4048, 4234, 3828, 3904, 4004, 4310, 4086, 4274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234979320 ingress_bytes=4462723072 link_bytes=4462723072 token_bytes=469843960 weight_movement_bytes=4227858432 total_nvlink_bytes=4697702392 weight_to_token_ratio=8.998431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498938637e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28784 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2043, 2059, 2116, 2063, 2062, 2021, 1992, 2028] payload_rows_per_dst=[4086, 4118, 4232, 4126, 4124, 4042, 3984, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4086, 4118, 4232, 4126, 4124, 4042, 3984, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235913216 dispatch_meta_bytes=115192 combine_payload_bytes=235913216 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236028408 ingress_bytes=1645199360 link_bytes=1645199360 token_bytes=471941624 weight_movement_bytes=1409286144 total_nvlink_bytes=1881227768 weight_to_token_ratio=2.986145 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212429676e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=232767488 dispatch_meta_bytes=113920 combine_payload_bytes=232767488 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232881408 ingress_bytes=2346696704 link_bytes=2346696704 token_bytes=465648896 weight_movement_bytes=2113929216 total_nvlink_bytes=2579578112 weight_to_token_ratio=4.539749 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314052198e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235798528 dispatch_meta_bytes=115136 combine_payload_bytes=235798528 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235913664 ingress_bytes=2349727744 link_bytes=2349727744 token_bytes=471712192 weight_movement_bytes=2113929216 total_nvlink_bytes=2585641408 weight_to_token_ratio=4.481396 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315749454e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 1996, 2100, 2016, 2020, 2037, 2050, 2091] payload_rows_per_dst=[4148, 3992, 4200, 4032, 4040, 4074, 4100, 4182] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 3992, 4200, 4032, 4040, 4074, 4100, 4182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32668 payload_rows_remote=28448 dup_rows=100 dup_rows_remote=86 dedup_ratio=0.003052 router_tokens_per_expert=[2069, 2059, 2089, 1980, 2004, 1988, 2089, 2106] payload_rows_per_dst=[4220, 4192, 4160, 4026, 4128, 4152, 3972, 3818] dup_rows_per_dst=[14, 32, 0, 18, 6, 30, 0, 0] combine_rows_per_src=[4220, 4192, 4160, 4026, 4128, 4152, 3972, 3818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28664 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2033, 2052, 2091, 2041, 2057, 1996, 2036, 2078] payload_rows_per_dst=[4066, 4104, 4182, 4082, 4114, 3992, 4072, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4066, 4104, 4182, 4082, 4114, 3992, 4072, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28780 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2019, 1994, 2108, 2017, 2067, 2085, 2039] payload_rows_per_dst=[4110, 4038, 3988, 4216, 4034, 4134, 4170, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4038, 3988, 4216, 4034, 4134, 4170, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2073, 2019, 2077, 2100, 2095, 1989, 1984, 2047] payload_rows_per_dst=[4146, 4038, 4154, 4200, 4190, 3978, 3968, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 4038, 4154, 4200, 4190, 3978, 3968, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28600 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2035, 1996, 2084, 1959, 2108, 2086, 2036] payload_rows_per_dst=[4160, 4070, 3992, 4168, 3918, 4216, 4172, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4070, 3992, 4168, 3918, 4216, 4172, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235176024 ingress_bytes=1644347392 link_bytes=1644347392 token_bytes=470237272 weight_movement_bytes=1409286144 total_nvlink_bytes=1879523416 weight_to_token_ratio=2.996968 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207659011e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233046016 dispatch_meta_bytes=114136 combine_payload_bytes=233046016 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233160152 ingress_bytes=3756261376 link_bytes=3756261376 token_bytes=466206168 weight_movement_bytes=3523215360 total_nvlink_bytes=3989421528 weight_to_token_ratio=7.557205 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103349576e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234815488 dispatch_meta_bytes=114656 combine_payload_bytes=234815488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234930144 ingress_bytes=1644101632 link_bytes=1644101632 token_bytes=469745632 weight_movement_bytes=1409286144 total_nvlink_bytes=1879031776 weight_to_token_ratio=3.000105 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206282857e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32696 payload_rows_remote=28630 dup_rows=72 dup_rows_remote=66 dedup_ratio=0.002197 router_tokens_per_expert=[2045, 2030, 2016, 2052, 2055, 2097, 2062, 2027] payload_rows_per_dst=[4018, 4010, 3942, 3910, 4348, 4208, 4194, 4066] dup_rows_per_dst=[0, 0, 0, 12, 42, 6, 6, 6] combine_rows_per_src=[4018, 4010, 3942, 3910, 4348, 4208, 4194, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28746 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2079, 2035, 2062, 2102, 1977, 2079, 2011, 2039] payload_rows_per_dst=[4158, 4070, 4124, 4204, 3954, 4158, 4022, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4158, 4070, 4124, 4204, 3954, 4158, 4022, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235765760 dispatch_meta_bytes=115120 combine_payload_bytes=235765760 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235880880 ingress_bytes=3758981120 link_bytes=3758981120 token_bytes=471646640 weight_movement_bytes=3523215360 total_nvlink_bytes=3994862000 weight_to_token_ratio=7.470032 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104872520e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114312 combine_payload_bytes=234110976 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234225288 ingress_bytes=3757326336 link_bytes=3757326336 token_bytes=468336264 weight_movement_bytes=3523215360 total_nvlink_bytes=3991551624 weight_to_token_ratio=7.522833 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103945910e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234291200 dispatch_meta_bytes=114400 combine_payload_bytes=234291200 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234405600 ingress_bytes=2348220416 link_bytes=2348220416 token_bytes=468696800 weight_movement_bytes=2113929216 total_nvlink_bytes=2582626016 weight_to_token_ratio=4.510228 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314905413e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114784 combine_payload_bytes=234536960 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234651744 ingress_bytes=3757752320 link_bytes=3757752320 token_bytes=469188704 weight_movement_bytes=3523215360 total_nvlink_bytes=3992404064 weight_to_token_ratio=7.509165 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104184443e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235487232 dispatch_meta_bytes=114984 combine_payload_bytes=235487232 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235602216 ingress_bytes=2349416448 link_bytes=2349416448 token_bytes=471089448 weight_movement_bytes=2113929216 total_nvlink_bytes=2585018664 weight_to_token_ratio=4.487320 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315575141e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2109, 2020, 2027, 2042, 2009, 2034, 2026, 2117] payload_rows_per_dst=[4218, 4040, 4054, 4084, 4018, 4068, 4052, 4234] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4218, 4040, 4054, 4084, 4018, 4068, 4052, 4234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32686 payload_rows_remote=28424 dup_rows=82 dup_rows_remote=46 dedup_ratio=0.002502 router_tokens_per_expert=[2002, 2033, 2069, 2071, 2076, 2002, 2120, 2011] payload_rows_per_dst=[4262, 4254, 4152, 3956, 3934, 3972, 4134, 4022] dup_rows_per_dst=[36, 36, 0, 10, 0, 0, 0, 0] combine_rows_per_src=[4262, 4254, 4152, 3956, 3934, 3972, 4134, 4022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2017, 2104, 2021, 2009, 2090, 1973, 2052, 2118] payload_rows_per_dst=[4034, 4208, 4042, 4018, 4180, 3946, 4104, 4236] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4034, 4208, 4042, 4018, 4180, 3946, 4104, 4236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28510 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2012, 2014, 2129, 2080, 2047, 1970, 2076, 2056] payload_rows_per_dst=[4024, 4028, 4258, 4160, 4094, 3940, 4152, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 4028, 4258, 4160, 4094, 3940, 4152, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1961, 2059, 2098, 2084, 2040, 2022, 2047, 2073] payload_rows_per_dst=[3922, 4118, 4196, 4168, 4080, 4044, 4094, 4146] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3922, 4118, 4196, 4168, 4080, 4044, 4094, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28610 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2096, 2004, 2027, 2079, 2010, 2062, 2056, 2050] payload_rows_per_dst=[4192, 4008, 4054, 4158, 4020, 4124, 4112, 4100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4192, 4008, 4054, 4158, 4020, 4124, 4112, 4100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235225200 ingress_bytes=1644396544 link_bytes=1644396544 token_bytes=470335600 weight_movement_bytes=1409286144 total_nvlink_bytes=1879621744 weight_to_token_ratio=2.996342 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207934242e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=232849408 dispatch_meta_bytes=113880 combine_payload_bytes=232849408 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=232963288 ingress_bytes=3756064768 link_bytes=3756064768 token_bytes=465812696 weight_movement_bytes=3523215360 total_nvlink_bytes=3989028056 weight_to_token_ratio=7.563588 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103239484e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114240 combine_payload_bytes=233963520 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234077760 ingress_bytes=3052535808 link_bytes=3052535808 token_bytes=468041280 weight_movement_bytes=2818572288 total_nvlink_bytes=3286613568 weight_to_token_ratio=6.022059 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709292633e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32658 payload_rows_remote=28376 dup_rows=110 dup_rows_remote=64 dedup_ratio=0.003357 router_tokens_per_expert=[2046, 2073, 2083, 2046, 2007, 2025, 2076, 2028] payload_rows_per_dst=[3770, 4244, 3844, 4092, 4372, 4022, 4032, 4282] dup_rows_per_dst=[0, 18, 0, 0, 46, 0, 0, 46] combine_rows_per_src=[3770, 4244, 3844, 4092, 4372, 4022, 4032, 4282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28674 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2028, 2047, 2058, 1970, 2086, 2047, 2099] payload_rows_per_dst=[4098, 4056, 4094, 4116, 3940, 4172, 4094, 4198] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4056, 4094, 4116, 3940, 4172, 4094, 4198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233553920 dispatch_meta_bytes=114040 combine_payload_bytes=233553920 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233667960 ingress_bytes=3052126208 link_bytes=3052126208 token_bytes=467221880 weight_movement_bytes=2818572288 total_nvlink_bytes=3285794168 weight_to_token_ratio=6.032620 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709063274e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234373120 dispatch_meta_bytes=114440 combine_payload_bytes=234373120 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234487560 ingress_bytes=2348302336 link_bytes=2348302336 token_bytes=468860680 weight_movement_bytes=2113929216 total_nvlink_bytes=2582789896 weight_to_token_ratio=4.508651 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314951285e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235126848 ingress_bytes=1644298240 link_bytes=1644298240 token_bytes=470138944 weight_movement_bytes=1409286144 total_nvlink_bytes=1879425088 weight_to_token_ratio=2.997595 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207383780e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=232456192 dispatch_meta_bytes=113760 combine_payload_bytes=232456192 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232569952 ingress_bytes=1641742336 link_bytes=1641742336 token_bytes=465026144 weight_movement_bytes=1409286144 total_nvlink_bytes=1874312288 weight_to_token_ratio=3.030553 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.193071785e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234897408 dispatch_meta_bytes=114696 combine_payload_bytes=234897408 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235012104 ingress_bytes=2348826624 link_bytes=2348826624 token_bytes=469909512 weight_movement_bytes=2113929216 total_nvlink_bytes=2583838728 weight_to_token_ratio=4.498588 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315244865e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2049, 2062, 2037, 1995, 2090, 2041, 2076] payload_rows_per_dst=[4068, 4098, 4124, 4074, 3990, 4180, 4082, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4098, 4124, 4074, 3990, 4180, 4082, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=28500 dup_rows=66 dup_rows_remote=48 dedup_ratio=0.002014 router_tokens_per_expert=[2024, 2087, 2094, 2032, 2080, 2081, 1979, 2007] payload_rows_per_dst=[4202, 4184, 4188, 3944, 4100, 4266, 4170, 3648] dup_rows_per_dst=[18, 0, 0, 2, 0, 24, 22, 0] combine_rows_per_src=[4202, 4184, 4188, 3944, 4100, 4266, 4170, 3648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2040, 2063, 2063, 2058, 2006, 2050, 2013] payload_rows_per_dst=[4182, 4080, 4126, 4126, 4116, 4012, 4100, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4080, 4126, 4126, 4116, 4012, 4100, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28612 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2031, 2023, 2078, 2073, 1977, 2053, 2088] payload_rows_per_dst=[4122, 4062, 4046, 4156, 4146, 3954, 4106, 4176] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4062, 4046, 4156, 4146, 3954, 4106, 4176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 1978, 2082, 2033, 2080, 2087, 2086, 2007] payload_rows_per_dst=[4062, 3956, 4164, 4066, 4160, 4174, 4172, 4014] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 3956, 4164, 4066, 4160, 4174, 4172, 4014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234192896 dispatch_meta_bytes=114352 combine_payload_bytes=234192896 prefetch_bytes=0 reduce_bytes=2113929216 egress_bytes=234307248 ingress_bytes=2348122112 link_bytes=2348122112 token_bytes=468500144 weight_movement_bytes=2113929216 total_nvlink_bytes=2582429360 weight_to_token_ratio=4.512121 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314850367e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114192 combine_payload_bytes=233472000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233586192 ingress_bytes=1642758144 link_bytes=1642758144 token_bytes=467058192 weight_movement_bytes=1409286144 total_nvlink_bytes=1876344336 weight_to_token_ratio=3.017367 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.198759885e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2009, 2105, 2085, 2095, 2056, 2021, 1964] payload_rows_per_dst=[4098, 4018, 4210, 4170, 4190, 4112, 4042, 3928] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4018, 4210, 4170, 4190, 4112, 4042, 3928] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32690 payload_rows_remote=28460 dup_rows=78 dup_rows_remote=48 dedup_ratio=0.002380 router_tokens_per_expert=[2061, 2038, 2050, 2075, 2041, 2036, 2046, 2037] payload_rows_per_dst=[4122, 3658, 4298, 4062, 4128, 4100, 4092, 4230] dup_rows_per_dst=[0, 0, 34, 0, 10, 4, 0, 30] combine_rows_per_src=[4122, 3658, 4298, 4062, 4128, 4100, 4092, 4230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235126848 ingress_bytes=1644298240 link_bytes=1644298240 token_bytes=470138944 weight_movement_bytes=1409286144 total_nvlink_bytes=1879425088 weight_to_token_ratio=2.997595 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207383780e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28870 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2084, 2054, 2048, 2037, 2034, 1949, 2112] payload_rows_per_dst=[4132, 4168, 4108, 4096, 4074, 4068, 3898, 4224] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4168, 4108, 4096, 4074, 4068, 3898, 4224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234389504 dispatch_meta_bytes=114448 combine_payload_bytes=234389504 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234503952 ingress_bytes=3052961792 link_bytes=3052961792 token_bytes=468893456 weight_movement_bytes=2818572288 total_nvlink_bytes=3287465744 weight_to_token_ratio=6.011115 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709531167e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234438384 ingress_bytes=2348253184 link_bytes=2348253184 token_bytes=468762352 weight_movement_bytes=2113929216 total_nvlink_bytes=2582691568 weight_to_token_ratio=4.509597 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314923762e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114312 combine_payload_bytes=234110976 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234225288 ingress_bytes=2348040192 link_bytes=2348040192 token_bytes=468336264 weight_movement_bytes=2113929216 total_nvlink_bytes=2582265480 weight_to_token_ratio=4.513700 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314804495e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28774 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2020, 2123, 2001, 2041, 2020, 1997, 2075, 2107] payload_rows_per_dst=[4040, 4246, 4002, 4082, 4040, 3994, 4150, 4214] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4040, 4246, 4002, 4082, 4040, 3994, 4150, 4214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=28580 dup_rows=24 dup_rows_remote=12 dedup_ratio=0.000732 router_tokens_per_expert=[2064, 2048, 2045, 2041, 2079, 2073, 2045, 1989] payload_rows_per_dst=[4164, 4148, 4026, 4152, 4136, 4050, 4090, 3978] dup_rows_per_dst=[12, 4, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[4164, 4148, 4026, 4152, 4136, 4050, 4090, 3978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2004, 2092, 1994, 2109, 2011, 2033, 2056, 2085] payload_rows_per_dst=[4008, 4184, 3988, 4218, 4022, 4066, 4112, 4170] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4008, 4184, 3988, 4218, 4022, 4066, 4112, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233144320 dispatch_meta_bytes=114032 combine_payload_bytes=233144320 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233258352 ingress_bytes=3051716608 link_bytes=3051716608 token_bytes=466402672 weight_movement_bytes=2818572288 total_nvlink_bytes=3284974960 weight_to_token_ratio=6.043216 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708833916e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=236503040 dispatch_meta_bytes=115480 combine_payload_bytes=236503040 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=236618520 ingress_bytes=1645789184 link_bytes=1645789184 token_bytes=473121560 weight_movement_bytes=1409286144 total_nvlink_bytes=1882407704 weight_to_token_ratio=2.978698 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.215732444e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235716608 dispatch_meta_bytes=115096 combine_payload_bytes=235716608 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235831704 ingress_bytes=1645002752 link_bytes=1645002752 token_bytes=471548312 weight_movement_bytes=1409286144 total_nvlink_bytes=1880834456 weight_to_token_ratio=2.988636 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211328753e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234127360 dispatch_meta_bytes=114368 combine_payload_bytes=234127360 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234241728 ingress_bytes=3052699648 link_bytes=3052699648 token_bytes=468369088 weight_movement_bytes=2818572288 total_nvlink_bytes=3286941376 weight_to_token_ratio=6.017844 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709384377e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234160128 dispatch_meta_bytes=114336 combine_payload_bytes=234160128 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234274464 ingress_bytes=3757375488 link_bytes=3757375488 token_bytes=468434592 weight_movement_bytes=3523215360 total_nvlink_bytes=3991649952 weight_to_token_ratio=7.521254 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103973433e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28668 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2127, 2051, 2020, 2050, 1982, 2041, 2054, 2059] payload_rows_per_dst=[4254, 4102, 4040, 4100, 3964, 4082, 4108, 4118] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4254, 4102, 4040, 4100, 3964, 4082, 4108, 4118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28562 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2040, 1953, 2039, 2103, 2024, 2134, 2030] payload_rows_per_dst=[4122, 4080, 3906, 4078, 4206, 4048, 4268, 4060] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4080, 3906, 4078, 4206, 4048, 4268, 4060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28570 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2001, 2093, 2099, 2065, 2021, 2122, 1975, 2008] payload_rows_per_dst=[4002, 4186, 4198, 4130, 4042, 4244, 3950, 4016] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4002, 4186, 4198, 4130, 4042, 4244, 3950, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2116, 1982, 2018, 2076, 2022, 2062, 2028, 2080] payload_rows_per_dst=[4232, 3964, 4036, 4152, 4044, 4124, 4056, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4232, 3964, 4036, 4152, 4044, 4124, 4056, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32634 payload_rows_remote=28254 dup_rows=134 dup_rows_remote=58 dedup_ratio=0.004089 router_tokens_per_expert=[2037, 2143, 2010, 2031, 2004, 2094, 2075, 1990] payload_rows_per_dst=[3678, 3914, 4090, 3912, 4124, 4204, 4332, 4380] dup_rows_per_dst=[0, 0, 8, 0, 14, 0, 36, 76] combine_rows_per_src=[3678, 3914, 4090, 3912, 4124, 4204, 4332, 4380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234848256 dispatch_meta_bytes=114672 combine_payload_bytes=234848256 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234962928 ingress_bytes=3053420544 link_bytes=3053420544 token_bytes=469811184 weight_movement_bytes=2818572288 total_nvlink_bytes=3288383472 weight_to_token_ratio=5.999372 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709788049e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233979904 dispatch_meta_bytes=114248 combine_payload_bytes=233979904 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234094152 ingress_bytes=3757195264 link_bytes=3757195264 token_bytes=468074056 weight_movement_bytes=3523215360 total_nvlink_bytes=3991289416 weight_to_token_ratio=7.527047 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103872515e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234045440 dispatch_meta_bytes=114280 combine_payload_bytes=234045440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234159720 ingress_bytes=1643331584 link_bytes=1643331584 token_bytes=468205160 weight_movement_bytes=1409286144 total_nvlink_bytes=1877491304 weight_to_token_ratio=3.009976 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201970910e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28726 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2027, 2080, 2066, 2106, 2044, 2021, 2021, 2019] payload_rows_per_dst=[4054, 4160, 4132, 4212, 4088, 4042, 4042, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4054, 4160, 4132, 4212, 4088, 4042, 4042, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=28536 dup_rows=64 dup_rows_remote=64 dedup_ratio=0.001953 router_tokens_per_expert=[2076, 2074, 1972, 2038, 2044, 2094, 2031, 2055] payload_rows_per_dst=[4168, 4266, 3948, 4176, 4088, 4054, 4182, 3822] dup_rows_per_dst=[0, 20, 2, 28, 0, 0, 14, 0] combine_rows_per_src=[4168, 4266, 3948, 4176, 4088, 4054, 4182, 3822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28500 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2134, 2075, 1988, 2029, 2080, 1966, 2032] payload_rows_per_dst=[4160, 4268, 4150, 3976, 4058, 4160, 3932, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4268, 4150, 3976, 4058, 4160, 3932, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235323552 ingress_bytes=1644494848 link_bytes=1644494848 token_bytes=470532256 weight_movement_bytes=1409286144 total_nvlink_bytes=1879818400 weight_to_token_ratio=2.995089 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208484703e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=231456768 dispatch_meta_bytes=113248 combine_payload_bytes=231456768 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231570016 ingress_bytes=1640742912 link_bytes=1640742912 token_bytes=463026784 weight_movement_bytes=1409286144 total_nvlink_bytes=1872312928 weight_to_token_ratio=3.043638 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.187475427e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28550 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2083, 2018, 2070, 2102, 2109, 2017, 2007, 1978] payload_rows_per_dst=[4166, 4036, 4140, 4204, 4218, 4034, 4014, 3956] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4166, 4036, 4140, 4204, 4218, 4034, 4014, 3956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2019, 2087, 2042, 2018, 2057, 2068, 2036] payload_rows_per_dst=[4114, 4038, 4174, 4084, 4036, 4114, 4136, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4038, 4174, 4084, 4036, 4114, 4136, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235323392 dispatch_meta_bytes=114904 combine_payload_bytes=235323392 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235438296 ingress_bytes=2349252608 link_bytes=2349252608 token_bytes=470761688 weight_movement_bytes=2113929216 total_nvlink_bytes=2584690904 weight_to_token_ratio=4.490444 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315483398e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=233766912 dispatch_meta_bytes=114400 combine_payload_bytes=233766912 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233881312 ingress_bytes=2347696128 link_bytes=2347696128 token_bytes=467648224 weight_movement_bytes=2113929216 total_nvlink_bytes=2581577440 weight_to_token_ratio=4.520341 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314611834e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114000 combine_payload_bytes=233472000 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233586000 ingress_bytes=3052044288 link_bytes=3052044288 token_bytes=467058000 weight_movement_bytes=2818572288 total_nvlink_bytes=3285630288 weight_to_token_ratio=6.034737 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709017403e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2086, 2073, 2026, 2061, 2036, 2079, 1999] payload_rows_per_dst=[4048, 4172, 4146, 4052, 4122, 4072, 4158, 3998] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4172, 4146, 4052, 4122, 4072, 4158, 3998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28804 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2109, 2055, 2083, 1999, 2008, 2045, 1982, 2103] payload_rows_per_dst=[4218, 4110, 4166, 3998, 4016, 4090, 3964, 4206] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4218, 4110, 4166, 3998, 4016, 4090, 3964, 4206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233881600 dispatch_meta_bytes=114200 combine_payload_bytes=233881600 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233995800 ingress_bytes=2347810816 link_bytes=2347810816 token_bytes=467877400 weight_movement_bytes=2113929216 total_nvlink_bytes=2581806616 weight_to_token_ratio=4.518126 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314676055e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114736 combine_payload_bytes=234979328 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235094064 ingress_bytes=1644265472 link_bytes=1644265472 token_bytes=470073392 weight_movement_bytes=1409286144 total_nvlink_bytes=1879359536 weight_to_token_ratio=2.998013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207200293e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=28656 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000732 router_tokens_per_expert=[2095, 2075, 2025, 1989, 2032, 2067, 2057, 2044] payload_rows_per_dst=[4086, 4150, 4006, 3978, 4166, 4134, 4136, 4088] dup_rows_per_dst=[0, 0, 0, 0, 20, 0, 4, 0] combine_rows_per_src=[4086, 4150, 4006, 3978, 4166, 4134, 4136, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114488 combine_payload_bytes=234471424 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234585912 ingress_bytes=3053043712 link_bytes=3053043712 token_bytes=469057336 weight_movement_bytes=2818572288 total_nvlink_bytes=3287629624 weight_to_token_ratio=6.009014 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709577038e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235962368 dispatch_meta_bytes=115216 combine_payload_bytes=235962368 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236077584 ingress_bytes=1645248512 link_bytes=1645248512 token_bytes=472039952 weight_movement_bytes=1409286144 total_nvlink_bytes=1881326096 weight_to_token_ratio=2.985523 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212704907e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2003, 2079, 2082, 2033, 2062, 2040, 2031, 2054] payload_rows_per_dst=[4006, 4158, 4164, 4066, 4124, 4080, 4062, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4006, 4158, 4164, 4066, 4124, 4080, 4062, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2107, 2049, 1996, 2063, 2059, 2006, 2050, 2054] payload_rows_per_dst=[4214, 4098, 3992, 4126, 4118, 4012, 4100, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4214, 4098, 3992, 4126, 4118, 4012, 4100, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32656 payload_rows_remote=28328 dup_rows=112 dup_rows_remote=46 dedup_ratio=0.003418 router_tokens_per_expert=[2013, 2068, 2126, 2113, 2005, 2010, 2036, 2013] payload_rows_per_dst=[4328, 4136, 4266, 4260, 3892, 4316, 3766, 3692] dup_rows_per_dst=[66, 0, 2, 6, 0, 38, 0, 0] combine_rows_per_src=[4328, 4136, 4266, 4260, 3892, 4316, 3766, 3692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234749952 dispatch_meta_bytes=114720 combine_payload_bytes=234749952 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=234864672 ingress_bytes=4462608384 link_bytes=4462608384 token_bytes=469614624 weight_movement_bytes=4227858432 total_nvlink_bytes=4697473056 weight_to_token_ratio=9.002825 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498874417e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2014, 2077, 2055, 2057, 2139, 1991, 2018, 2033] payload_rows_per_dst=[4028, 4154, 4110, 4114, 4278, 3982, 4036, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4028, 4154, 4110, 4114, 4278, 3982, 4036, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28678 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1976, 1956, 2062, 2135, 2045, 2061, 2023, 2126] payload_rows_per_dst=[3952, 3912, 4124, 4270, 4090, 4122, 4046, 4252] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3952, 3912, 4124, 4270, 4090, 4122, 4046, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235126848 ingress_bytes=3053584384 link_bytes=3053584384 token_bytes=470138944 weight_movement_bytes=2818572288 total_nvlink_bytes=3288711232 weight_to_token_ratio=5.995190 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709879792e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234979320 ingress_bytes=1644150784 link_bytes=1644150784 token_bytes=469843960 weight_movement_bytes=1409286144 total_nvlink_bytes=1879130104 weight_to_token_ratio=2.999477 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206558088e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=232062976 dispatch_meta_bytes=113496 combine_payload_bytes=232062976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232176472 ingress_bytes=1641349120 link_bytes=1641349120 token_bytes=464239448 weight_movement_bytes=1409286144 total_nvlink_bytes=1873525592 weight_to_token_ratio=3.035688 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.190869939e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2005, 2008, 2096, 2021, 2055, 2081, 2062, 2056] payload_rows_per_dst=[4010, 4016, 4192, 4042, 4110, 4162, 4124, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4010, 4016, 4192, 4042, 4110, 4162, 4124, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2026, 2024, 2111, 2009, 2089, 2001, 2090, 2034] payload_rows_per_dst=[4052, 4048, 4222, 4018, 4178, 4002, 4180, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4052, 4048, 4222, 4018, 4178, 4002, 4180, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234848184 ingress_bytes=2348662784 link_bytes=2348662784 token_bytes=469581752 weight_movement_bytes=2113929216 total_nvlink_bytes=2583510968 weight_to_token_ratio=4.501728 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315153121e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234930176 dispatch_meta_bytes=114712 combine_payload_bytes=234930176 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235044888 ingress_bytes=1644216320 link_bytes=1644216320 token_bytes=469975064 weight_movement_bytes=1409286144 total_nvlink_bytes=1879261208 weight_to_token_ratio=2.998640 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206925062e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=28388 dup_rows=66 dup_rows_remote=30 dedup_ratio=0.002014 router_tokens_per_expert=[2081, 2026, 2011, 2038, 2020, 2042, 2081, 2085] payload_rows_per_dst=[4162, 4046, 3902, 3966, 4022, 4118, 4172, 4314] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 24, 36] combine_rows_per_src=[4162, 4046, 3902, 3966, 4022, 4118, 4172, 4314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234094592 dispatch_meta_bytes=114304 combine_payload_bytes=234094592 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234208896 ingress_bytes=2348023808 link_bytes=2348023808 token_bytes=468303488 weight_movement_bytes=2113929216 total_nvlink_bytes=2582232704 weight_to_token_ratio=4.514016 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314795321e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234192896 dispatch_meta_bytes=114352 combine_payload_bytes=234192896 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234307248 ingress_bytes=3052765184 link_bytes=3052765184 token_bytes=468500144 weight_movement_bytes=2818572288 total_nvlink_bytes=3287072432 weight_to_token_ratio=6.016161 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709421074e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28796 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2043, 2061, 2022, 2059, 2024, 1986, 2108, 2081] payload_rows_per_dst=[4086, 4122, 4044, 4118, 4048, 3972, 4216, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4086, 4122, 4044, 4118, 4048, 3972, 4216, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2017, 2017, 2012, 2043, 2073, 2083, 2084, 2055] payload_rows_per_dst=[4034, 4034, 4024, 4086, 4146, 4166, 4168, 4110] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4034, 4034, 4024, 4086, 4146, 4166, 4168, 4110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=232554496 dispatch_meta_bytes=113672 combine_payload_bytes=232554496 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232668168 ingress_bytes=2346483712 link_bytes=2346483712 token_bytes=465222664 weight_movement_bytes=2113929216 total_nvlink_bytes=2579151880 weight_to_token_ratio=4.543908 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313932932e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32682 payload_rows_remote=28508 dup_rows=86 dup_rows_remote=82 dedup_ratio=0.002625 router_tokens_per_expert=[2069, 2057, 2126, 2035, 2097, 2022, 1963, 2015] payload_rows_per_dst=[4174, 4448, 4288, 4128, 4078, 3658, 3878, 4030] dup_rows_per_dst=[4, 68, 6, 8, 0, 0, 0, 0] combine_rows_per_src=[4174, 4448, 4288, 4128, 4078, 3658, 3878, 4030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1977, 2064, 2004, 2029, 2060, 2084, 2112, 2054] payload_rows_per_dst=[3954, 4128, 4008, 4058, 4120, 4168, 4224, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3954, 4128, 4008, 4058, 4120, 4168, 4224, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28628 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2032, 2048, 2070, 2093, 2093, 1980, 2027] payload_rows_per_dst=[4082, 4064, 4096, 4140, 4186, 4186, 3960, 4054] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4064, 4096, 4140, 4186, 4186, 3960, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235896832 dispatch_meta_bytes=115184 combine_payload_bytes=235896832 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236012016 ingress_bytes=1645182976 link_bytes=1645182976 token_bytes=471908848 weight_movement_bytes=1409286144 total_nvlink_bytes=1881194992 weight_to_token_ratio=2.986352 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212337932e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=235503864 ingress_bytes=940032000 link_bytes=940032000 token_bytes=470892792 weight_movement_bytes=704643072 total_nvlink_bytes=1175535864 weight_to_token_ratio=1.496398 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.263786811e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2050, 2095, 2071, 2042, 2000, 1991, 2106] payload_rows_per_dst=[4058, 4100, 4190, 4142, 4084, 4000, 3982, 4212] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4100, 4190, 4142, 4084, 4000, 3982, 4212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2078, 2114, 2041, 2073, 1981, 2038, 2043] payload_rows_per_dst=[4032, 4156, 4228, 4082, 4146, 3962, 4076, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4156, 4228, 4082, 4146, 3962, 4076, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234799008 ingress_bytes=4462542848 link_bytes=4462542848 token_bytes=469483424 weight_movement_bytes=4227858432 total_nvlink_bytes=4697341856 weight_to_token_ratio=9.005341 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498837719e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233537536 dispatch_meta_bytes=114360 combine_payload_bytes=233537536 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233651896 ingress_bytes=3052109824 link_bytes=3052109824 token_bytes=467189432 weight_movement_bytes=2818572288 total_nvlink_bytes=3285761720 weight_to_token_ratio=6.033039 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709054100e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234520576 dispatch_meta_bytes=114512 combine_payload_bytes=234520576 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234635088 ingress_bytes=2348449792 link_bytes=2348449792 token_bytes=469155664 weight_movement_bytes=2113929216 total_nvlink_bytes=2583084880 weight_to_token_ratio=4.505816 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315033854e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32664 payload_rows_remote=28544 dup_rows=104 dup_rows_remote=94 dedup_ratio=0.003174 router_tokens_per_expert=[2082, 2042, 2104, 2014, 2033, 2019, 2065, 2025] payload_rows_per_dst=[4098, 3718, 4034, 3882, 4224, 4414, 4174, 4120] dup_rows_per_dst=[0, 0, 0, 4, 10, 72, 8, 10] combine_rows_per_src=[4098, 3718, 4034, 3882, 4224, 4414, 4174, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114312 combine_payload_bytes=234110976 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234225288 ingress_bytes=3757326336 link_bytes=3757326336 token_bytes=468336264 weight_movement_bytes=3523215360 total_nvlink_bytes=3991551624 weight_to_token_ratio=7.522833 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103945910e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235159632 ingress_bytes=3053617152 link_bytes=3053617152 token_bytes=470204496 weight_movement_bytes=2818572288 total_nvlink_bytes=3288776784 weight_to_token_ratio=5.994354 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709898141e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28612 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2008, 2006, 2080, 2000, 2078, 2112, 2079] payload_rows_per_dst=[4042, 4016, 4012, 4160, 4000, 4156, 4224, 4158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4016, 4012, 4160, 4000, 4156, 4224, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2048, 2093, 2067, 2061, 1983, 2030, 2069, 2033] payload_rows_per_dst=[4096, 4186, 4134, 4122, 3966, 4060, 4138, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 4186, 4134, 4122, 3966, 4060, 4138, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233832448 dispatch_meta_bytes=114552 combine_payload_bytes=233832448 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233947000 ingress_bytes=1643118592 link_bytes=1643118592 token_bytes=467779448 weight_movement_bytes=1409286144 total_nvlink_bytes=1877065592 weight_to_token_ratio=3.012715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.200778244e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2078, 1948, 2067, 2040, 2046, 2050, 2097] payload_rows_per_dst=[4116, 4156, 3896, 4134, 4080, 4092, 4100, 4194] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4156, 3896, 4134, 4080, 4092, 4100, 4194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234389504 dispatch_meta_bytes=114448 combine_payload_bytes=234389504 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234503952 ingress_bytes=2348318720 link_bytes=2348318720 token_bytes=468893456 weight_movement_bytes=2113929216 total_nvlink_bytes=2582822672 weight_to_token_ratio=4.508336 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314960460e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32696 payload_rows_remote=28332 dup_rows=72 dup_rows_remote=42 dedup_ratio=0.002197 router_tokens_per_expert=[2098, 2065, 2049, 2043, 1995, 2048, 1974, 2112] payload_rows_per_dst=[4364, 4134, 4098, 4282, 3998, 4084, 3850, 3886] dup_rows_per_dst=[30, 2, 0, 24, 0, 16, 0, 0] combine_rows_per_src=[4364, 4134, 4098, 4282, 3998, 4084, 3850, 3886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2141, 1985, 2057, 1990, 2030, 2076, 2071] payload_rows_per_dst=[4068, 4282, 3970, 4114, 3980, 4060, 4152, 4142] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4282, 3970, 4114, 3980, 4060, 4152, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234143744 dispatch_meta_bytes=114328 combine_payload_bytes=234143744 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234258072 ingress_bytes=3757359104 link_bytes=3757359104 token_bytes=468401816 weight_movement_bytes=3523215360 total_nvlink_bytes=3991617176 weight_to_token_ratio=7.521780 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103964258e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2062, 1978, 2037, 2054, 2039, 2084, 2105, 2025] payload_rows_per_dst=[4124, 3956, 4074, 4108, 4078, 4168, 4210, 4050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4124, 3956, 4074, 4108, 4078, 4168, 4210, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235126848 ingress_bytes=3053584384 link_bytes=3053584384 token_bytes=470138944 weight_movement_bytes=2818572288 total_nvlink_bytes=3288711232 weight_to_token_ratio=5.995190 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709879792e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2084, 2053, 2051, 2050, 1990, 2077, 2035, 2044] payload_rows_per_dst=[4168, 4106, 4102, 4100, 3980, 4154, 4070, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4168, 4106, 4102, 4100, 3980, 4154, 4070, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232095744 dispatch_meta_bytes=113496 combine_payload_bytes=232095744 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232209240 ingress_bytes=3050668032 link_bytes=3050668032 token_bytes=464304984 weight_movement_bytes=2818572288 total_nvlink_bytes=3282877272 weight_to_token_ratio=6.070519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708246757e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234848184 ingress_bytes=1644019712 link_bytes=1644019712 token_bytes=469581752 weight_movement_bytes=1409286144 total_nvlink_bytes=1878867896 weight_to_token_ratio=3.001152 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205824140e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32640 payload_rows_remote=28518 dup_rows=128 dup_rows_remote=128 dedup_ratio=0.003906 router_tokens_per_expert=[2083, 2085, 2043, 2017, 2022, 2010, 2064, 2060] payload_rows_per_dst=[3760, 3742, 4102, 4052, 4592, 4084, 4186, 4122] dup_rows_per_dst=[0, 0, 12, 2, 96, 10, 8, 0] combine_rows_per_src=[3760, 3742, 4102, 4052, 4592, 4084, 4186, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235176024 ingress_bytes=2348990464 link_bytes=2348990464 token_bytes=470237272 weight_movement_bytes=2113929216 total_nvlink_bytes=2584166488 weight_to_token_ratio=4.495452 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315336608e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28668 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1932, 2089, 2081, 2039, 2051, 2050, 2098, 2044] payload_rows_per_dst=[3864, 4178, 4162, 4078, 4102, 4100, 4196, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3864, 4178, 4162, 4078, 4102, 4100, 4196, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235208808 ingress_bytes=2349023232 link_bytes=2349023232 token_bytes=470302824 weight_movement_bytes=2113929216 total_nvlink_bytes=2584232040 weight_to_token_ratio=4.494826 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315354957e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2055, 2018, 2012, 2010, 2059, 2056, 2136] payload_rows_per_dst=[4076, 4110, 4036, 4024, 4020, 4118, 4112, 4272] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4110, 4036, 4024, 4020, 4118, 4112, 4272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233619456 dispatch_meta_bytes=114584 combine_payload_bytes=233619456 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=233734040 ingress_bytes=4461477888 link_bytes=4461477888 token_bytes=467353496 weight_movement_bytes=4227858432 total_nvlink_bytes=4695211928 weight_to_token_ratio=9.046382 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498241386e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28772 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2125, 2070, 2081, 2016, 1998, 2013, 2060, 2021] payload_rows_per_dst=[4250, 4140, 4162, 4032, 3996, 4026, 4120, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4250, 4140, 4162, 4032, 3996, 4026, 4120, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234848256 dispatch_meta_bytes=114672 combine_payload_bytes=234848256 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234962928 ingress_bytes=2348777472 link_bytes=2348777472 token_bytes=469811184 weight_movement_bytes=2113929216 total_nvlink_bytes=2583740400 weight_to_token_ratio=4.499529 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315217342e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32680 payload_rows_remote=28556 dup_rows=88 dup_rows_remote=80 dedup_ratio=0.002686 router_tokens_per_expert=[2038, 2016, 2113, 2005, 2053, 2042, 2090, 2027] payload_rows_per_dst=[4124, 4196, 3986, 4102, 4006, 4058, 4314, 3894] dup_rows_per_dst=[8, 38, 0, 16, 0, 0, 26, 0] combine_rows_per_src=[4124, 4196, 3986, 4102, 4006, 4058, 4314, 3894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234880968 ingress_bytes=3757981696 link_bytes=3757981696 token_bytes=469647304 weight_movement_bytes=3523215360 total_nvlink_bytes=3992862664 weight_to_token_ratio=7.501832 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104312884e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2008, 2079, 2093, 2017, 2045, 2041, 2073, 2028] payload_rows_per_dst=[4016, 4158, 4186, 4034, 4090, 4082, 4146, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4016, 4158, 4186, 4034, 4090, 4082, 4146, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2094, 2037, 2062, 2030, 2002, 2058, 2037, 2064] payload_rows_per_dst=[4188, 4074, 4124, 4060, 4004, 4116, 4074, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4188, 4074, 4124, 4060, 4004, 4116, 4074, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235700224 dispatch_meta_bytes=115088 combine_payload_bytes=235700224 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235815312 ingress_bytes=1644986368 link_bytes=1644986368 token_bytes=471515536 weight_movement_bytes=1409286144 total_nvlink_bytes=1880801680 weight_to_token_ratio=2.988843 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211237010e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28828 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2067, 2035, 2120, 2038, 2019, 2049, 1970, 2086] payload_rows_per_dst=[4134, 4070, 4240, 4076, 4038, 4098, 3940, 4172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4134, 4070, 4240, 4076, 4038, 4098, 3940, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114544 combine_payload_bytes=233930752 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=234045296 ingress_bytes=938573824 link_bytes=938573824 token_bytes=467976048 weight_movement_bytes=704643072 total_nvlink_bytes=1172619120 weight_to_token_ratio=1.505725 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.255621634e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32746 payload_rows_remote=28638 dup_rows=22 dup_rows_remote=14 dedup_ratio=0.000671 router_tokens_per_expert=[2048, 2068, 2062, 2109, 2041, 1990, 2043, 2023] payload_rows_per_dst=[4096, 4136, 4104, 4162, 4074, 3972, 4094, 4108] dup_rows_per_dst=[0, 0, 0, 0, 14, 0, 0, 8] combine_rows_per_src=[4096, 4136, 4104, 4162, 4074, 3972, 4094, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235503864 ingress_bytes=1644675072 link_bytes=1644675072 token_bytes=470892792 weight_movement_bytes=1409286144 total_nvlink_bytes=1880178936 weight_to_token_ratio=2.992796 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209493882e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234766224 ingress_bytes=2348580864 link_bytes=2348580864 token_bytes=469417872 weight_movement_bytes=2113929216 total_nvlink_bytes=2583347088 weight_to_token_ratio=4.503299 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315107249e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28682 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2140, 2001, 1983, 2011, 2070, 2043, 2043, 2093] payload_rows_per_dst=[4280, 4002, 3966, 4022, 4140, 4086, 4086, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4280, 4002, 3966, 4022, 4140, 4086, 4086, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=236158976 dispatch_meta_bytes=115312 combine_payload_bytes=236158976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236274288 ingress_bytes=1645445120 link_bytes=1645445120 token_bytes=472433264 weight_movement_bytes=1409286144 total_nvlink_bytes=1881719408 weight_to_token_ratio=2.983038 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213805829e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2019, 2080, 2048, 2064, 2065, 2018, 2034] payload_rows_per_dst=[4112, 4038, 4160, 4096, 4128, 4130, 4036, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4038, 4160, 4096, 4128, 4130, 4036, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1998, 2041, 2027, 2030, 2082, 2074, 2057, 2075] payload_rows_per_dst=[3996, 4082, 4054, 4060, 4164, 4148, 4114, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3996, 4082, 4054, 4060, 4164, 4148, 4114, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114608 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234717104 ingress_bytes=2348531712 link_bytes=2348531712 token_bytes=469319600 weight_movement_bytes=2113929216 total_nvlink_bytes=2583248816 weight_to_token_ratio=4.504242 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315079726e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234962944 dispatch_meta_bytes=114728 combine_payload_bytes=234962944 prefetch_bytes=0 reduce_bytes=2818572288 egress_bytes=235077672 ingress_bytes=3053535232 link_bytes=3053535232 token_bytes=470040616 weight_movement_bytes=2818572288 total_nvlink_bytes=3288612904 weight_to_token_ratio=5.996444 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709852269e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32644 payload_rows_remote=28466 dup_rows=124 dup_rows_remote=106 dedup_ratio=0.003784 router_tokens_per_expert=[2006, 2121, 2018, 1997, 2078, 2004, 2057, 2103] payload_rows_per_dst=[4178, 4324, 4198, 4060, 4100, 4008, 4030, 3746] dup_rows_per_dst=[18, 14, 30, 22, 6, 0, 34, 0] combine_rows_per_src=[4178, 4324, 4198, 4060, 4100, 4008, 4030, 3746] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=114920 combine_payload_bytes=235356160 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235471080 ingress_bytes=1644642304 link_bytes=1644642304 token_bytes=470827240 weight_movement_bytes=1409286144 total_nvlink_bytes=1880113384 weight_to_token_ratio=2.993213 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209310395e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2086, 2031, 2006, 2049, 2052, 2022, 2092, 2046] payload_rows_per_dst=[4172, 4062, 4012, 4098, 4104, 4044, 4184, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4172, 4062, 4012, 4098, 4104, 4044, 4184, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234438384 ingress_bytes=3052896256 link_bytes=3052896256 token_bytes=468762352 weight_movement_bytes=2818572288 total_nvlink_bytes=3287334640 weight_to_token_ratio=6.012796 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709494469e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2001, 2024, 2036, 2100, 2057, 2097, 2006, 2063] payload_rows_per_dst=[4002, 4048, 4072, 4200, 4114, 4194, 4012, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4002, 4048, 4072, 4200, 4114, 4194, 4012, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1992, 2035, 2044, 2059, 2047, 2033, 2028, 2146] payload_rows_per_dst=[3984, 4070, 4088, 4118, 4094, 4066, 4056, 4292] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3984, 4070, 4088, 4118, 4094, 4066, 4056, 4292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233193472 dispatch_meta_bytes=114288 combine_payload_bytes=233193472 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233307760 ingress_bytes=1642479616 link_bytes=1642479616 token_bytes=466501232 weight_movement_bytes=1409286144 total_nvlink_bytes=1875787376 weight_to_token_ratio=3.020970 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.197200245e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234979320 ingress_bytes=1644150784 link_bytes=1644150784 token_bytes=469843960 weight_movement_bytes=1409286144 total_nvlink_bytes=1879130104 weight_to_token_ratio=2.999477 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206558088e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32762 payload_rows_remote=28690 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000183 router_tokens_per_expert=[2054, 2012, 2033, 2091, 2068, 2083, 2012, 2031] payload_rows_per_dst=[4108, 3982, 4026, 4182, 4150, 4216, 4026, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[4108, 3982, 4026, 4182, 4150, 4216, 4026, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=235192416 ingress_bytes=4462936064 link_bytes=4462936064 token_bytes=470270048 weight_movement_bytes=4227858432 total_nvlink_bytes=4698128480 weight_to_token_ratio=8.990278 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499057904e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2104, 2020, 2057, 2068, 1987, 2057, 2058, 2033] payload_rows_per_dst=[4208, 4040, 4114, 4136, 3974, 4114, 4116, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4208, 4040, 4114, 4136, 3974, 4114, 4116, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235323552 ingress_bytes=3758424064 link_bytes=3758424064 token_bytes=470532256 weight_movement_bytes=3523215360 total_nvlink_bytes=3993747616 weight_to_token_ratio=7.487723 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104560591e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28806 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 1981, 2031, 2015, 2053, 2089, 2024, 2136] payload_rows_per_dst=[4110, 3962, 4062, 4030, 4106, 4178, 4048, 4272] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 3962, 4062, 4030, 4106, 4178, 4048, 4272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28556 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1993, 2074, 2090, 1961, 2106, 2053, 2079, 2028] payload_rows_per_dst=[3986, 4148, 4180, 3922, 4212, 4106, 4158, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3986, 4148, 4180, 3922, 4212, 4106, 4158, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235028480 dispatch_meta_bytes=114784 combine_payload_bytes=235028480 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=235143264 ingress_bytes=4462886912 link_bytes=4462886912 token_bytes=470171744 weight_movement_bytes=4227858432 total_nvlink_bytes=4698030176 weight_to_token_ratio=8.992158 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499030381e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234848184 ingress_bytes=1644019712 link_bytes=1644019712 token_bytes=469581752 weight_movement_bytes=1409286144 total_nvlink_bytes=1878867896 weight_to_token_ratio=3.001152 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205824140e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235978752 dispatch_meta_bytes=115224 combine_payload_bytes=235978752 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=236093976 ingress_bytes=3054551040 link_bytes=3054551040 token_bytes=472072728 weight_movement_bytes=2818572288 total_nvlink_bytes=3290645016 weight_to_token_ratio=5.970631 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710421079e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28628 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2025, 2090, 2070, 2019, 2090, 2007, 2029] payload_rows_per_dst=[4108, 4050, 4180, 4140, 4038, 4180, 4014, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4050, 4180, 4140, 4038, 4180, 4014, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=28450 dup_rows=92 dup_rows_remote=66 dedup_ratio=0.002808 router_tokens_per_expert=[2059, 2083, 2037, 1982, 2044, 2054, 2046, 2079] payload_rows_per_dst=[4226, 4256, 4164, 4068, 3872, 3942, 4136, 4012] dup_rows_per_dst=[26, 20, 16, 28, 0, 0, 2, 0] combine_rows_per_src=[4226, 4256, 4164, 4068, 3872, 3942, 4136, 4012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28710 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2002, 2083, 2029, 2100, 2011, 2015, 2058, 2086] payload_rows_per_dst=[4004, 4166, 4058, 4200, 4022, 4030, 4116, 4172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4004, 4166, 4058, 4200, 4022, 4030, 4116, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2058, 2072, 2034, 2058, 2050, 2017, 2039] payload_rows_per_dst=[4112, 4116, 4144, 4068, 4116, 4100, 4034, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4116, 4144, 4068, 4116, 4100, 4034, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114224 combine_payload_bytes=233930752 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234044976 ingress_bytes=2347859968 link_bytes=2347859968 token_bytes=467975728 weight_movement_bytes=2113929216 total_nvlink_bytes=2581904944 weight_to_token_ratio=4.517177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314703578e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32742 payload_rows_remote=28506 dup_rows=26 dup_rows_remote=22 dedup_ratio=0.000793 router_tokens_per_expert=[2043, 2004, 2000, 2000, 2043, 2130, 2065, 2099] payload_rows_per_dst=[4086, 4008, 4000, 3882, 4152, 4212, 4166, 4236] dup_rows_per_dst=[0, 0, 0, 0, 16, 0, 6, 4] combine_rows_per_src=[4086, 4008, 4000, 3882, 4152, 4212, 4166, 4236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234520576 dispatch_meta_bytes=114512 combine_payload_bytes=234520576 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234635088 ingress_bytes=3757735936 link_bytes=3757735936 token_bytes=469155664 weight_movement_bytes=3523215360 total_nvlink_bytes=3992371024 weight_to_token_ratio=7.509694 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104175269e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233062400 dispatch_meta_bytes=114064 combine_payload_bytes=233062400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233176464 ingress_bytes=1642348544 link_bytes=1642348544 token_bytes=466238864 weight_movement_bytes=1409286144 total_nvlink_bytes=1875525008 weight_to_token_ratio=3.022670 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196466296e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28616 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2018, 1983, 2017, 2089, 2072, 2076, 2050, 2079] payload_rows_per_dst=[4036, 3966, 4034, 4178, 4144, 4152, 4100, 4158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4036, 3966, 4034, 4178, 4144, 4152, 4100, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235503864 ingress_bytes=1644675072 link_bytes=1644675072 token_bytes=470892792 weight_movement_bytes=1409286144 total_nvlink_bytes=1880178936 weight_to_token_ratio=2.992796 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209493882e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235192320 dispatch_meta_bytes=114840 combine_payload_bytes=235192320 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235307160 ingress_bytes=3053764608 link_bytes=3053764608 token_bytes=470499480 weight_movement_bytes=2818572288 total_nvlink_bytes=3289071768 weight_to_token_ratio=5.990596 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709980710e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28702 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2003, 2033, 2076, 2008, 2025, 2117, 2047, 2075] payload_rows_per_dst=[4006, 4066, 4152, 4016, 4050, 4234, 4094, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4006, 4066, 4152, 4016, 4050, 4234, 4094, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233521152 dispatch_meta_bytes=114112 combine_payload_bytes=233521152 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233635264 ingress_bytes=3052093440 link_bytes=3052093440 token_bytes=467156416 weight_movement_bytes=2818572288 total_nvlink_bytes=3285728704 weight_to_token_ratio=6.033466 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709044926e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7515012384 prefetch_bytes=19025362944 reduce_bytes=15502147584 weight_movement_bytes=34527510528 egress_bytes=3758423328 ingress_bytes=38284099584 link_bytes=38284099584 total_nvlink_bytes=42042522912 weight_to_token_ratio=4.594472 prefetch_share_of_ingress=0.496952 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=22 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.339843607e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7467392840 prefetch_bytes=19025362944 reduce_bytes=18320719872 weight_movement_bytes=37346082816 egress_bytes=3734609736 ingress_bytes=41078865920 link_bytes=41078865920 total_nvlink_bytes=44813475656 weight_to_token_ratio=5.001221 prefetch_share_of_ingress=0.463142 dedup_ratio=0.002460 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=26 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.437653137e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234422272 dispatch_meta_bytes=114464 combine_payload_bytes=234422272 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=234536736 ingress_bytes=5166923776 link_bytes=5166923776 token_bytes=468959008 weight_movement_bytes=4932501504 total_nvlink_bytes=5401460512 weight_to_token_ratio=10.517980 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893261637e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2088, 2002, 2134, 2081, 1939, 2077, 2021] payload_rows_per_dst=[4084, 4176, 4004, 4268, 4162, 3878, 4154, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4176, 4004, 4268, 4162, 3878, 4154, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7531269280 prefetch_bytes=19025362944 reduce_bytes=15502147584 weight_movement_bytes=34527510528 egress_bytes=3766553760 ingress_bytes=38292226048 link_bytes=38292226048 total_nvlink_bytes=42058779808 weight_to_token_ratio=4.584554 prefetch_share_of_ingress=0.496847 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=22 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.340128012e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235126784 dispatch_meta_bytes=114808 combine_payload_bytes=235126784 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235241592 ingress_bytes=2349056000 link_bytes=2349056000 token_bytes=470368376 weight_movement_bytes=2113929216 total_nvlink_bytes=2584297592 weight_to_token_ratio=4.494199 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315373306e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7472406808 prefetch_bytes=19025362944 reduce_bytes=22548578304 weight_movement_bytes=41573941248 egress_bytes=3737116952 ingress_bytes=45309231104 link_bytes=45309231104 total_nvlink_bytes=49046348056 weight_to_token_ratio=5.563661 prefetch_share_of_ingress=0.419900 dedup_ratio=0.002140 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=32 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.585704882e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7519928784 prefetch_bytes=16911433728 reduce_bytes=15502147584 weight_movement_bytes=32413581312 egress_bytes=3760882128 ingress_bytes=36172627968 link_bytes=36172627968 total_nvlink_bytes=39933510096 weight_to_token_ratio=4.310357 prefetch_share_of_ingress=0.467520 dedup_ratio=0.000000 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=22 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.265947609e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2055, 2037, 2060, 2109, 2035, 2031, 2015] payload_rows_per_dst=[4084, 4110, 4074, 4120, 4218, 4070, 4062, 4030] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4110, 4074, 4120, 4218, 4070, 4062, 4030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114424 combine_payload_bytes=234340352 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234454776 ingress_bytes=2348269568 link_bytes=2348269568 token_bytes=468795128 weight_movement_bytes=2113929216 total_nvlink_bytes=2582724344 weight_to_token_ratio=4.509282 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314932936e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7514389640 prefetch_bytes=19730006016 reduce_bytes=16911433728 weight_movement_bytes=36641439744 egress_bytes=3758111880 ingress_bytes=40397717504 link_bytes=40397717504 total_nvlink_bytes=44155829384 weight_to_token_ratio=4.876170 prefetch_share_of_ingress=0.488394 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=24 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.413814720e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235176024 ingress_bytes=1644347392 link_bytes=1644347392 token_bytes=470237272 weight_movement_bytes=1409286144 total_nvlink_bytes=1879523416 weight_to_token_ratio=2.996968 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207659011e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7510227088 prefetch_bytes=19025362944 reduce_bytes=23957864448 weight_movement_bytes=42983227392 egress_bytes=3756030096 ingress_bytes=46737424384 link_bytes=46737424384 total_nvlink_bytes=50493454480 weight_to_token_ratio=5.723293 prefetch_share_of_ingress=0.407069 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=34 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.635687921e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7513340808 prefetch_bytes=19025362944 reduce_bytes=22548578304 weight_movement_bytes=41573941248 egress_bytes=3757587336 ingress_bytes=45329694720 link_bytes=45329694720 total_nvlink_bytes=49087282056 weight_to_token_ratio=5.533350 prefetch_share_of_ingress=0.419711 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=32 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.586421056e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none - [2026-09-16 21:35:51] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 576.6 | throughput per GPU (TFLOP/s/GPU): 578.2 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 9.682616E+00 | loss scale: 1.0 | grad norm: 57.383 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32680 payload_rows_remote=28416 dup_rows=88 dup_rows_remote=60 dedup_ratio=0.002686 router_tokens_per_expert=[2067, 2025, 2013, 2065, 2060, 2064, 1994, 2096] payload_rows_per_dst=[4264, 4166, 4032, 4222, 4056, 4120, 3922, 3898] dup_rows_per_dst=[28, 40, 2, 12, 0, 0, 6, 0] combine_rows_per_src=[4264, 4166, 4032, 4222, 4056, 4120, 3922, 3898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28690 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1989, 2093, 2057, 2012, 2039, 2025, 2061, 2108] payload_rows_per_dst=[3978, 4186, 4114, 4024, 4078, 4050, 4122, 4216] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3978, 4186, 4114, 4024, 4078, 4050, 4122, 4216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2009, 2067, 2091, 2041, 2016, 2079, 2046, 2035] payload_rows_per_dst=[4018, 4134, 4182, 4082, 4032, 4158, 4092, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4018, 4134, 4182, 4082, 4032, 4158, 4092, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28804 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2081, 2070, 1982, 2033, 2022, 2129, 2071, 1996] payload_rows_per_dst=[4162, 4140, 3964, 4066, 4044, 4258, 4142, 3992] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4162, 4140, 3964, 4066, 4044, 4258, 4142, 3992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 2084, 2140, 2046, 2080, 1982, 1995, 2026] payload_rows_per_dst=[4062, 4168, 4280, 4092, 4160, 3964, 3990, 4052] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 4168, 4280, 4092, 4160, 3964, 3990, 4052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235028480 dispatch_meta_bytes=114760 combine_payload_bytes=235028480 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235143240 ingress_bytes=2348957696 link_bytes=2348957696 token_bytes=470171720 weight_movement_bytes=2113929216 total_nvlink_bytes=2584100936 weight_to_token_ratio=4.496079 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315318259e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=232783872 dispatch_meta_bytes=113904 combine_payload_bytes=232783872 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232897776 ingress_bytes=2346713088 link_bytes=2346713088 token_bytes=465681648 weight_movement_bytes=2113929216 total_nvlink_bytes=2579610864 weight_to_token_ratio=4.539430 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314061373e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28706 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1995, 2031, 2094, 2050, 1998, 2056, 2086, 2074] payload_rows_per_dst=[3990, 4062, 4188, 4100, 3996, 4112, 4172, 4148] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3990, 4062, 4188, 4100, 3996, 4112, 4172, 4148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=28502 dup_rows=60 dup_rows_remote=38 dedup_ratio=0.001831 router_tokens_per_expert=[2100, 2034, 2027, 2027, 2091, 1977, 2097, 2031] payload_rows_per_dst=[4184, 3902, 3918, 3932, 4320, 4050, 4196, 4206] dup_rows_per_dst=[0, 0, 0, 0, 14, 20, 4, 22] combine_rows_per_src=[4184, 3902, 3918, 3932, 4320, 4050, 4196, 4206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 1985, 2044, 2119, 2034, 2018, 2061, 2082] payload_rows_per_dst=[4082, 3970, 4088, 4238, 4068, 4036, 4122, 4164] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 3970, 4088, 4238, 4068, 4036, 4122, 4164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234913792 dispatch_meta_bytes=114704 combine_payload_bytes=234913792 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235028496 ingress_bytes=2348843008 link_bytes=2348843008 token_bytes=469942288 weight_movement_bytes=2113929216 total_nvlink_bytes=2583871504 weight_to_token_ratio=4.498274 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315254039e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235962368 dispatch_meta_bytes=115216 combine_payload_bytes=235962368 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236077584 ingress_bytes=3054534656 link_bytes=3054534656 token_bytes=472039952 weight_movement_bytes=2818572288 total_nvlink_bytes=3290612240 weight_to_token_ratio=5.971046 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710411905e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234913792 dispatch_meta_bytes=114704 combine_payload_bytes=234913792 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235028496 ingress_bytes=2348843008 link_bytes=2348843008 token_bytes=469942288 weight_movement_bytes=2113929216 total_nvlink_bytes=2583871504 weight_to_token_ratio=4.498274 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315254039e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235159552 dispatch_meta_bytes=114824 combine_payload_bytes=235159552 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235274376 ingress_bytes=3053731840 link_bytes=3053731840 token_bytes=470433928 weight_movement_bytes=2818572288 total_nvlink_bytes=3289006216 weight_to_token_ratio=5.991431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709962361e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233488384 dispatch_meta_bytes=114160 combine_payload_bytes=233488384 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233602544 ingress_bytes=3756703744 link_bytes=3756703744 token_bytes=467090928 weight_movement_bytes=3523215360 total_nvlink_bytes=3990306288 weight_to_token_ratio=7.542890 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103597284e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235487472 ingress_bytes=2349301760 link_bytes=2349301760 token_bytes=470860016 weight_movement_bytes=2113929216 total_nvlink_bytes=2584789232 weight_to_token_ratio=4.489507 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315510921e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28826 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2023, 2046, 2088, 2119, 1971, 2036, 2080, 2021] payload_rows_per_dst=[4046, 4092, 4176, 4238, 3942, 4072, 4160, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4046, 4092, 4176, 4238, 3942, 4072, 4160, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=28550 dup_rows=92 dup_rows_remote=88 dedup_ratio=0.002808 router_tokens_per_expert=[2040, 2049, 1989, 2032, 2096, 2083, 2070, 2025] payload_rows_per_dst=[4126, 4068, 4470, 3644, 4012, 4166, 4140, 4050] dup_rows_per_dst=[4, 2, 86, 0, 0, 0, 0, 0] combine_rows_per_src=[4126, 4068, 4470, 3644, 4012, 4166, 4140, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=236142592 dispatch_meta_bytes=115304 combine_payload_bytes=236142592 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=236257896 ingress_bytes=3054714880 link_bytes=3054714880 token_bytes=472400488 weight_movement_bytes=2818572288 total_nvlink_bytes=3290972776 weight_to_token_ratio=5.966489 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710512823e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233881600 dispatch_meta_bytes=114552 combine_payload_bytes=233881600 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233996152 ingress_bytes=3052453888 link_bytes=3052453888 token_bytes=467877752 weight_movement_bytes=2818572288 total_nvlink_bytes=3286450040 weight_to_token_ratio=6.024164 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709246762e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2093, 2065, 2028, 2058, 2053, 2094, 2018, 1975] payload_rows_per_dst=[4186, 4130, 4056, 4116, 4106, 4188, 4036, 3950] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4186, 4130, 4056, 4116, 4106, 4188, 4036, 3950] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=28290 dup_rows=98 dup_rows_remote=42 dedup_ratio=0.002991 router_tokens_per_expert=[2105, 1975, 1969, 2068, 2051, 2060, 2113, 2043] payload_rows_per_dst=[4158, 3900, 3938, 3908, 4136, 4076, 4174, 4380] dup_rows_per_dst=[0, 0, 0, 0, 4, 12, 26, 56] combine_rows_per_src=[4158, 3900, 3938, 3908, 4136, 4076, 4174, 4380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28512 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2002, 1994, 1995, 2128, 2081, 2064, 2070, 2050] payload_rows_per_dst=[4004, 3988, 3990, 4256, 4162, 4128, 4140, 4100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4004, 3988, 3990, 4256, 4162, 4128, 4140, 4100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2115, 2006, 2086, 2046, 2036, 1989, 2069] payload_rows_per_dst=[4074, 4230, 4012, 4172, 4092, 4072, 3978, 4138] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4230, 4012, 4172, 4092, 4072, 3978, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1962, 2057, 2029, 2129, 2113, 2010, 2030, 2054] payload_rows_per_dst=[3924, 4114, 4058, 4258, 4226, 4020, 4060, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3924, 4114, 4058, 4258, 4226, 4020, 4060, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2123, 2122, 1991, 2088, 2044, 2017, 2027, 1972] payload_rows_per_dst=[4246, 4244, 3982, 4176, 4088, 4034, 4054, 3944] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4246, 4244, 3982, 4176, 4088, 4034, 4054, 3944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28592 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2106, 1988, 2037, 2088, 2088, 2068, 2023, 1986] payload_rows_per_dst=[4212, 3976, 4074, 4176, 4176, 4136, 4046, 3972] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4212, 3976, 4074, 4176, 4176, 4136, 4046, 3972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32696 payload_rows_remote=28436 dup_rows=72 dup_rows_remote=52 dedup_ratio=0.002197 router_tokens_per_expert=[2075, 2069, 2078, 2138, 1978, 2018, 2005, 2023] payload_rows_per_dst=[4260, 4154, 4432, 4124, 3956, 3830, 3894, 4046] dup_rows_per_dst=[20, 2, 50, 0, 0, 0, 0, 0] combine_rows_per_src=[4260, 4154, 4432, 4124, 3956, 3830, 3894, 4046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235487472 ingress_bytes=2349301760 link_bytes=2349301760 token_bytes=470860016 weight_movement_bytes=2113929216 total_nvlink_bytes=2584789232 weight_to_token_ratio=4.489507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315510921e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=231751680 dispatch_meta_bytes=113328 combine_payload_bytes=231751680 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231865008 ingress_bytes=1641037824 link_bytes=1641037824 token_bytes=463616688 weight_movement_bytes=1409286144 total_nvlink_bytes=1872902832 weight_to_token_ratio=3.039766 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.189126812e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233570304 dispatch_meta_bytes=114048 combine_payload_bytes=233570304 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233684352 ingress_bytes=4461428736 link_bytes=4461428736 token_bytes=467254656 weight_movement_bytes=4227858432 total_nvlink_bytes=4695113088 weight_to_token_ratio=9.048296 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498213863e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235684176 ingress_bytes=1644855296 link_bytes=1644855296 token_bytes=471253328 weight_movement_bytes=1409286144 total_nvlink_bytes=1880539472 weight_to_token_ratio=2.990507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210503061e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234848184 ingress_bytes=3053305856 link_bytes=3053305856 token_bytes=469581752 weight_movement_bytes=2818572288 total_nvlink_bytes=3288154040 weight_to_token_ratio=6.002304 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709723828e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235503864 ingress_bytes=2349318144 link_bytes=2349318144 token_bytes=470892792 weight_movement_bytes=2113929216 total_nvlink_bytes=2584822008 weight_to_token_ratio=4.489194 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315520095e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234225664 dispatch_meta_bytes=114368 combine_payload_bytes=234225664 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234340032 ingress_bytes=1643511808 link_bytes=1643511808 token_bytes=468565696 weight_movement_bytes=1409286144 total_nvlink_bytes=1877851840 weight_to_token_ratio=3.007660 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202980089e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=232947712 dispatch_meta_bytes=113952 combine_payload_bytes=232947712 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233061664 ingress_bytes=1642233856 link_bytes=1642233856 token_bytes=466009376 weight_movement_bytes=1409286144 total_nvlink_bytes=1875295520 weight_to_token_ratio=3.024158 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195824091e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=28390 dup_rows=96 dup_rows_remote=38 dedup_ratio=0.002930 router_tokens_per_expert=[2095, 2054, 2037, 2063, 1999, 2065, 2074, 1997] payload_rows_per_dst=[4190, 4038, 3756, 4030, 3992, 4210, 4174, 4282] dup_rows_per_dst=[0, 0, 0, 2, 8, 18, 10, 58] combine_rows_per_src=[4190, 4038, 3756, 4030, 3992, 4210, 4174, 4282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28556 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1965, 2065, 2059, 2014, 2067, 2067, 2106, 2041] payload_rows_per_dst=[3930, 4130, 4118, 4028, 4134, 4134, 4212, 4082] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3930, 4130, 4118, 4028, 4134, 4134, 4212, 4082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2010, 2094, 2087, 2036, 2129, 2004, 2041, 1983] payload_rows_per_dst=[4020, 4188, 4174, 4072, 4258, 4008, 4082, 3966] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4020, 4188, 4174, 4072, 4258, 4008, 4082, 3966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1998, 2095, 2058, 2003, 2029, 2070, 2035, 2096] payload_rows_per_dst=[3996, 4190, 4116, 4006, 4058, 4140, 4070, 4192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3996, 4190, 4116, 4006, 4058, 4140, 4070, 4192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2105, 2024, 1980, 2112, 2023, 2068, 2021, 2051] payload_rows_per_dst=[4210, 4048, 3960, 4224, 4046, 4136, 4042, 4102] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4210, 4048, 3960, 4224, 4046, 4136, 4042, 4102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1965, 2021, 2044, 2053, 2070, 2078, 2119, 2034] payload_rows_per_dst=[3930, 4042, 4088, 4106, 4140, 4156, 4238, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3930, 4042, 4088, 4106, 4140, 4156, 4238, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32602 payload_rows_remote=28360 dup_rows=166 dup_rows_remote=150 dedup_ratio=0.005066 router_tokens_per_expert=[2059, 2064, 1987, 2049, 2013, 2027, 2109, 2076] payload_rows_per_dst=[4242, 4250, 4248, 4096, 4190, 4090, 3508, 3978] dup_rows_per_dst=[16, 26, 54, 22, 28, 20, 0, 0] combine_rows_per_src=[4242, 4250, 4248, 4096, 4190, 4090, 3508, 3978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28668 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1965, 2031, 2041, 2035, 2050, 2143, 2067, 2052] payload_rows_per_dst=[3930, 4062, 4082, 4070, 4100, 4286, 4134, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3930, 4062, 4082, 4070, 4100, 4286, 4134, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=232570880 dispatch_meta_bytes=113712 combine_payload_bytes=232570880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232684592 ingress_bytes=1641857024 link_bytes=1641857024 token_bytes=465255472 weight_movement_bytes=1409286144 total_nvlink_bytes=1874541616 weight_to_token_ratio=3.029059 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.193713990e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114224 combine_payload_bytes=233930752 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234044976 ingress_bytes=2347859968 link_bytes=2347859968 token_bytes=467975728 weight_movement_bytes=2113929216 total_nvlink_bytes=2581904944 weight_to_token_ratio=4.517177 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314703578e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235192416 ingress_bytes=3758292992 link_bytes=3758292992 token_bytes=470270048 weight_movement_bytes=3523215360 total_nvlink_bytes=3993485408 weight_to_token_ratio=7.491898 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104487197e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234667872 ingress_bytes=2348482560 link_bytes=2348482560 token_bytes=469221216 weight_movement_bytes=2113929216 total_nvlink_bytes=2583150432 weight_to_token_ratio=4.505187 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315052203e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114312 combine_payload_bytes=234110976 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234225288 ingress_bytes=3052683264 link_bytes=3052683264 token_bytes=468336264 weight_movement_bytes=2818572288 total_nvlink_bytes=3286908552 weight_to_token_ratio=6.018266 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709375203e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235061280 ingress_bytes=2348875776 link_bytes=2348875776 token_bytes=470007840 weight_movement_bytes=2113929216 total_nvlink_bytes=2583937056 weight_to_token_ratio=4.497647 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315272388e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=232325120 dispatch_meta_bytes=114040 combine_payload_bytes=232325120 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232439160 ingress_bytes=1641611264 link_bytes=1641611264 token_bytes=464764280 weight_movement_bytes=1409286144 total_nvlink_bytes=1874050424 weight_to_token_ratio=3.032260 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.192337836e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234848256 dispatch_meta_bytes=114672 combine_payload_bytes=234848256 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234962928 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=469811184 weight_movement_bytes=1409286144 total_nvlink_bytes=1879097328 weight_to_token_ratio=2.999686 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32748 payload_rows_remote=28762 dup_rows=20 dup_rows_remote=2 dedup_ratio=0.000610 router_tokens_per_expert=[2048, 2030, 2105, 2023, 2136, 2017, 2094, 1931] payload_rows_per_dst=[4096, 4060, 4210, 4046, 4272, 3898, 4180, 3986] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 18] combine_rows_per_src=[4096, 4060, 4210, 4046, 4272, 3898, 4180, 3986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1992, 2011, 1995, 2068, 1978, 2091, 2141, 2108] payload_rows_per_dst=[3984, 4022, 3990, 4136, 3956, 4182, 4282, 4216] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3984, 4022, 3990, 4136, 3956, 4182, 4282, 4216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28410 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2087, 1975, 2039, 1992, 2010, 2179, 2053] payload_rows_per_dst=[4098, 4174, 3950, 4078, 3984, 4020, 4358, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4174, 3950, 4078, 3984, 4020, 4358, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28666 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2043, 2003, 2049, 2015, 2051, 2112, 2046] payload_rows_per_dst=[4130, 4086, 4006, 4098, 4030, 4102, 4224, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4086, 4006, 4098, 4030, 4102, 4224, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2116, 2036, 2042, 2022, 2050, 2025, 2029, 2064] payload_rows_per_dst=[4232, 4072, 4084, 4044, 4100, 4050, 4058, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4232, 4072, 4084, 4044, 4100, 4050, 4058, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28624 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2020, 2008, 2072, 2089, 2054, 2060, 2011, 2070] payload_rows_per_dst=[4040, 4016, 4144, 4178, 4108, 4120, 4022, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4040, 4016, 4144, 4178, 4108, 4120, 4022, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=28344 dup_rows=76 dup_rows_remote=56 dedup_ratio=0.002319 router_tokens_per_expert=[2119, 2081, 2012, 2025, 2035, 2004, 2046, 2062] payload_rows_per_dst=[4348, 4100, 4046, 4128, 4250, 4008, 4092, 3720] dup_rows_per_dst=[20, 0, 12, 10, 34, 0, 0, 0] combine_rows_per_src=[4348, 4100, 4046, 4128, 4250, 4008, 4092, 3720] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 2, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28808 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2033, 2060, 2020, 2089, 1980, 2073, 2037, 2092] payload_rows_per_dst=[4066, 4120, 4040, 4178, 3960, 4146, 4074, 4184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4066, 4120, 4040, 4178, 3960, 4146, 4074, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235618304 dispatch_meta_bytes=115056 combine_payload_bytes=235618304 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235733360 ingress_bytes=3054190592 link_bytes=3054190592 token_bytes=471351664 weight_movement_bytes=2818572288 total_nvlink_bytes=3289923952 weight_to_token_ratio=5.979765 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710219243e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234667872 ingress_bytes=2348482560 link_bytes=2348482560 token_bytes=469221216 weight_movement_bytes=2113929216 total_nvlink_bytes=2583150432 weight_to_token_ratio=4.505187 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315052203e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=232734720 dispatch_meta_bytes=113640 combine_payload_bytes=232734720 prefetch_bytes=704643072 reduce_bytes=5637144576 egress_bytes=232848360 ingress_bytes=6574522368 link_bytes=6574522368 token_bytes=465583080 weight_movement_bytes=6341787648 total_nvlink_bytes=6807370728 weight_to_token_ratio=13.621173 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.681458092e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234831872 dispatch_meta_bytes=114664 combine_payload_bytes=234831872 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234946536 ingress_bytes=3053404160 link_bytes=3053404160 token_bytes=469778408 weight_movement_bytes=2818572288 total_nvlink_bytes=3288350696 weight_to_token_ratio=5.999791 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709778874e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=232194048 dispatch_meta_bytes=113600 combine_payload_bytes=232194048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232307648 ingress_bytes=1641480192 link_bytes=1641480192 token_bytes=464501696 weight_movement_bytes=1409286144 total_nvlink_bytes=1873787840 weight_to_token_ratio=3.033974 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.191603888e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235192416 ingress_bytes=1644363776 link_bytes=1644363776 token_bytes=470270048 weight_movement_bytes=1409286144 total_nvlink_bytes=1879556192 weight_to_token_ratio=2.996759 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207750754e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234487808 dispatch_meta_bytes=114496 combine_payload_bytes=234487808 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234602304 ingress_bytes=1643773952 link_bytes=1643773952 token_bytes=469090112 weight_movement_bytes=1409286144 total_nvlink_bytes=1878376256 weight_to_token_ratio=3.004297 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204447986e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235995136 dispatch_meta_bytes=115232 combine_payload_bytes=235995136 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236110368 ingress_bytes=3054567424 link_bytes=3054567424 token_bytes=472105504 weight_movement_bytes=2818572288 total_nvlink_bytes=3290677792 weight_to_token_ratio=5.970217 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710430254e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32658 payload_rows_remote=28490 dup_rows=110 dup_rows_remote=104 dedup_ratio=0.003357 router_tokens_per_expert=[2027, 2092, 2069, 1969, 2091, 2015, 2044, 2077] payload_rows_per_dst=[4054, 3808, 4128, 4028, 3788, 4170, 4514, 4168] dup_rows_per_dst=[0, 0, 0, 6, 0, 28, 70, 6] combine_rows_per_src=[4054, 3808, 4128, 4028, 3788, 4170, 4514, 4168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2043, 2094, 2094, 2038, 2086, 2036, 1992, 2001] payload_rows_per_dst=[4086, 4188, 4188, 4076, 4172, 4072, 3984, 4002] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4086, 4188, 4188, 4076, 4172, 4072, 3984, 4002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32728 payload_rows_remote=28584 dup_rows=40 dup_rows_remote=38 dedup_ratio=0.001221 router_tokens_per_expert=[2055, 2080, 2107, 2029, 2041, 2026, 1970, 2076] payload_rows_per_dst=[4144, 4146, 4420, 4044, 4072, 4052, 3940, 3910] dup_rows_per_dst=[2, 4, 24, 10, 0, 0, 0, 0] combine_rows_per_src=[4144, 4146, 4420, 4044, 4072, 4052, 3940, 3910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1933, 2056, 2089, 2037, 2093, 2092, 1986, 2098] payload_rows_per_dst=[3866, 4112, 4178, 4074, 4186, 4184, 3972, 4196] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3866, 4112, 4178, 4074, 4186, 4184, 3972, 4196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28808 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2091, 2004, 2043, 2055, 2056, 1980, 2113] payload_rows_per_dst=[4084, 4182, 4008, 4086, 4110, 4112, 3960, 4226] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4182, 4008, 4086, 4110, 4112, 3960, 4226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233390080 dispatch_meta_bytes=114376 combine_payload_bytes=233390080 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233504456 ingress_bytes=4461248512 link_bytes=4461248512 token_bytes=466894536 weight_movement_bytes=4227858432 total_nvlink_bytes=4694752968 weight_to_token_ratio=9.055275 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498112945e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 2026, 2102, 2058, 2064, 2002, 1997, 2075] payload_rows_per_dst=[4120, 4052, 4204, 4116, 4128, 4004, 3994, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 4052, 4204, 4116, 4128, 4004, 3994, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28804 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2062, 2103, 1982, 2033, 2070, 2022, 2054, 2058] payload_rows_per_dst=[4124, 4206, 3964, 4066, 4140, 4044, 4108, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4124, 4206, 3964, 4066, 4140, 4044, 4108, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28616 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2013, 1992, 1999, 2107, 2076, 2058, 2094, 2045] payload_rows_per_dst=[4026, 3984, 3998, 4214, 4152, 4116, 4188, 4090] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4026, 3984, 3998, 4214, 4152, 4116, 4188, 4090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235159632 ingress_bytes=1644331008 link_bytes=1644331008 token_bytes=470204496 weight_movement_bytes=1409286144 total_nvlink_bytes=1879490640 weight_to_token_ratio=2.997177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207567267e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234160128 dispatch_meta_bytes=114488 combine_payload_bytes=234160128 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234274616 ingress_bytes=2348089344 link_bytes=2348089344 token_bytes=468434744 weight_movement_bytes=2113929216 total_nvlink_bytes=2582363960 weight_to_token_ratio=4.512751 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314832019e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234160128 dispatch_meta_bytes=114336 combine_payload_bytes=234160128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234274464 ingress_bytes=1643446272 link_bytes=1643446272 token_bytes=468434592 weight_movement_bytes=1409286144 total_nvlink_bytes=1877720736 weight_to_token_ratio=3.008501 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202613115e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235995136 dispatch_meta_bytes=115232 combine_payload_bytes=235995136 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236110368 ingress_bytes=1645281280 link_bytes=1645281280 token_bytes=472105504 weight_movement_bytes=1409286144 total_nvlink_bytes=1881391648 weight_to_token_ratio=2.985108 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212888394e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=235356336 ingress_bytes=4463099904 link_bytes=4463099904 token_bytes=470597808 weight_movement_bytes=4227858432 total_nvlink_bytes=4698456240 weight_to_token_ratio=8.984016 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499149647e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235962368 dispatch_meta_bytes=115216 combine_payload_bytes=235962368 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=236077584 ingress_bytes=3054534656 link_bytes=3054534656 token_bytes=472039952 weight_movement_bytes=2818572288 total_nvlink_bytes=3290612240 weight_to_token_ratio=5.971046 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710411905e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234422272 dispatch_meta_bytes=114464 combine_payload_bytes=234422272 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234536736 ingress_bytes=2348351488 link_bytes=2348351488 token_bytes=468959008 weight_movement_bytes=2113929216 total_nvlink_bytes=2582888224 weight_to_token_ratio=4.507706 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314978808e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28544 dup_rows=84 dup_rows_remote=82 dedup_ratio=0.002563 router_tokens_per_expert=[2092, 2013, 2020, 2082, 1999, 2065, 2048, 2065] payload_rows_per_dst=[3966, 4026, 3806, 3892, 4226, 4310, 4318, 4140] dup_rows_per_dst=[0, 0, 0, 0, 30, 22, 30, 2] combine_rows_per_src=[3966, 4026, 3806, 3892, 4226, 4310, 4318, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28806 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2086, 2060, 2066, 1981, 2081, 2000, 2038, 2072] payload_rows_per_dst=[4172, 4120, 4132, 3962, 4162, 4000, 4076, 4144] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4172, 4120, 4132, 3962, 4162, 4000, 4076, 4144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32668 payload_rows_remote=28148 dup_rows=100 dup_rows_remote=36 dedup_ratio=0.003052 router_tokens_per_expert=[2026, 2032, 2119, 2056, 2031, 2049, 2009, 2062] payload_rows_per_dst=[4520, 3932, 4076, 4118, 4062, 3836, 4018, 4106] dup_rows_per_dst=[64, 32, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[4520, 3932, 4076, 4118, 4062, 3836, 4018, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2092, 2059, 2047, 2064, 1976, 2057, 2057, 2032] payload_rows_per_dst=[4184, 4118, 4094, 4128, 3952, 4114, 4114, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4184, 4118, 4094, 4128, 3952, 4114, 4114, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28608 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2125, 2035, 2032, 2057, 2044, 1971, 2080, 2040] payload_rows_per_dst=[4250, 4070, 4064, 4114, 4088, 3942, 4160, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4250, 4070, 4064, 4114, 4088, 3942, 4160, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233832448 dispatch_meta_bytes=114504 combine_payload_bytes=233832448 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233946952 ingress_bytes=3757047808 link_bytes=3757047808 token_bytes=467779400 weight_movement_bytes=3523215360 total_nvlink_bytes=3990994760 weight_to_token_ratio=7.531788 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103789946e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28504 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2004, 2132, 2053, 2094, 2018, 1972, 1990, 2121] payload_rows_per_dst=[4008, 4264, 4106, 4188, 4036, 3944, 3980, 4242] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4008, 4264, 4106, 4188, 4036, 3944, 3980, 4242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2008, 2018, 2062, 2108, 2025, 2134, 1981, 2048] payload_rows_per_dst=[4016, 4036, 4124, 4216, 4050, 4268, 3962, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4016, 4036, 4124, 4216, 4050, 4268, 3962, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2040, 2012, 2060, 2058, 2058, 2112, 2051, 1993] payload_rows_per_dst=[4080, 4024, 4120, 4116, 4116, 4224, 4102, 3986] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4080, 4024, 4120, 4116, 4116, 4224, 4102, 3986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235978752 dispatch_meta_bytes=115224 combine_payload_bytes=235978752 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236093976 ingress_bytes=3054551040 link_bytes=3054551040 token_bytes=472072728 weight_movement_bytes=2818572288 total_nvlink_bytes=3290645016 weight_to_token_ratio=5.970631 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710421079e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=230588416 dispatch_meta_bytes=112736 combine_payload_bytes=230588416 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=230701152 ingress_bytes=1639874560 link_bytes=1639874560 token_bytes=461289568 weight_movement_bytes=1409286144 total_nvlink_bytes=1870575712 weight_to_token_ratio=3.055101 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.182613019e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234848184 ingress_bytes=2348662784 link_bytes=2348662784 token_bytes=469581752 weight_movement_bytes=2113929216 total_nvlink_bytes=2583510968 weight_to_token_ratio=4.501728 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315153121e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234356736 dispatch_meta_bytes=114432 combine_payload_bytes=234356736 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234471168 ingress_bytes=1643642880 link_bytes=1643642880 token_bytes=468827904 weight_movement_bytes=1409286144 total_nvlink_bytes=1878114048 weight_to_token_ratio=3.005978 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203714038e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233504768 dispatch_meta_bytes=114016 combine_payload_bytes=233504768 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233618784 ingress_bytes=2347433984 link_bytes=2347433984 token_bytes=467123552 weight_movement_bytes=2113929216 total_nvlink_bytes=2581052768 weight_to_token_ratio=4.525418 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314465044e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234766224 ingress_bytes=2348580864 link_bytes=2348580864 token_bytes=469417872 weight_movement_bytes=2113929216 total_nvlink_bytes=2583347088 weight_to_token_ratio=4.503299 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315107249e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234831792 ingress_bytes=2348646400 link_bytes=2348646400 token_bytes=469548976 weight_movement_bytes=2113929216 total_nvlink_bytes=2583478192 weight_to_token_ratio=4.502042 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315143947e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=28428 dup_rows=58 dup_rows_remote=32 dedup_ratio=0.001770 router_tokens_per_expert=[1921, 2137, 2053, 2088, 2065, 2080, 1969, 2071] payload_rows_per_dst=[3842, 4274, 3818, 4106, 4064, 4192, 4132, 4282] dup_rows_per_dst=[0, 0, 0, 0, 8, 2, 22, 26] combine_rows_per_src=[3842, 4274, 3818, 4106, 4064, 4192, 4132, 4282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1966, 2045, 2056, 2081, 2037, 2066, 2096, 2037] payload_rows_per_dst=[3932, 4090, 4112, 4162, 4074, 4132, 4192, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3932, 4090, 4112, 4162, 4074, 4132, 4192, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32694 payload_rows_remote=28658 dup_rows=74 dup_rows_remote=54 dedup_ratio=0.002258 router_tokens_per_expert=[1977, 2064, 2047, 2046, 2016, 2098, 2064, 2072] payload_rows_per_dst=[4036, 4146, 4086, 4486, 4026, 3802, 3982, 4130] dup_rows_per_dst=[20, 2, 2, 50, 0, 0, 0, 0] combine_rows_per_src=[4036, 4146, 4086, 4486, 4026, 3802, 3982, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2050, 2076, 2062, 2036, 2115, 1999, 2023, 2023] payload_rows_per_dst=[4100, 4152, 4124, 4072, 4230, 3998, 4046, 4046] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4100, 4152, 4124, 4072, 4230, 3998, 4046, 4046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28674 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2008, 2081, 2044, 2019, 2072, 2047, 2077, 2036] payload_rows_per_dst=[4016, 4162, 4088, 4038, 4144, 4094, 4154, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4016, 4162, 4088, 4038, 4144, 4094, 4154, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=232882176 dispatch_meta_bytes=113840 combine_payload_bytes=232882176 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=232996016 ingress_bytes=2346811392 link_bytes=2346811392 token_bytes=465878192 weight_movement_bytes=2113929216 total_nvlink_bytes=2579807408 weight_to_token_ratio=4.537515 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314116419e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114424 combine_payload_bytes=234340352 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234454776 ingress_bytes=2348269568 link_bytes=2348269568 token_bytes=468795128 weight_movement_bytes=2113929216 total_nvlink_bytes=2582724344 weight_to_token_ratio=4.509282 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314932936e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2098, 2023, 2060, 2053, 1997, 2048, 2013, 2092] payload_rows_per_dst=[4196, 4046, 4120, 4106, 3994, 4096, 4026, 4184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4196, 4046, 4120, 4106, 3994, 4096, 4026, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2120, 2082, 1986, 2026, 2075, 2008, 2016] payload_rows_per_dst=[4142, 4240, 4164, 3972, 4052, 4150, 4016, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4240, 4164, 3972, 4052, 4150, 4016, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28458 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2009, 2028, 2155, 2023, 2038, 2097, 2038, 1996] payload_rows_per_dst=[4018, 4056, 4310, 4046, 4076, 4194, 4076, 3992] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4018, 4056, 4310, 4046, 4076, 4194, 4076, 3992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114848 combine_payload_bytes=234766336 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234881184 ingress_bytes=2348695552 link_bytes=2348695552 token_bytes=469647520 weight_movement_bytes=2113929216 total_nvlink_bytes=2583576736 weight_to_token_ratio=4.501097 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315171470e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235405512 ingress_bytes=1644576768 link_bytes=1644576768 token_bytes=470696136 weight_movement_bytes=1409286144 total_nvlink_bytes=1879982280 weight_to_token_ratio=2.994047 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208943421e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234897408 dispatch_meta_bytes=114696 combine_payload_bytes=234897408 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235012104 ingress_bytes=3053469696 link_bytes=3053469696 token_bytes=469909512 weight_movement_bytes=2818572288 total_nvlink_bytes=3288481800 weight_to_token_ratio=5.998117 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709815572e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235405512 ingress_bytes=2349219840 link_bytes=2349219840 token_bytes=470696136 weight_movement_bytes=2113929216 total_nvlink_bytes=2584625352 weight_to_token_ratio=4.491070 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315465049e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235356336 ingress_bytes=2349170688 link_bytes=2349170688 token_bytes=470597808 weight_movement_bytes=2113929216 total_nvlink_bytes=2584527024 weight_to_token_ratio=4.492008 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315437526e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233127936 dispatch_meta_bytes=113832 combine_payload_bytes=233127936 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=233241768 ingress_bytes=4460986368 link_bytes=4460986368 token_bytes=466369704 weight_movement_bytes=4227858432 total_nvlink_bytes=4694228136 weight_to_token_ratio=9.065465 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.497966155e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28820 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2048, 2052, 1974, 2086, 2074, 2072, 2027] payload_rows_per_dst=[4102, 4096, 4104, 3948, 4172, 4148, 4144, 4054] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4096, 4104, 3948, 4172, 4148, 4144, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32724 payload_rows_remote=28462 dup_rows=44 dup_rows_remote=18 dedup_ratio=0.001343 router_tokens_per_expert=[2018, 2013, 2126, 2014, 2020, 2111, 2042, 2040] payload_rows_per_dst=[4006, 3766, 4196, 4028, 4094, 4220, 4152, 4262] dup_rows_per_dst=[0, 0, 0, 0, 8, 0, 10, 26] combine_rows_per_src=[4006, 3766, 4196, 4028, 4094, 4220, 4152, 4262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32696 payload_rows_remote=28502 dup_rows=72 dup_rows_remote=50 dedup_ratio=0.002197 router_tokens_per_expert=[2050, 2014, 2124, 2065, 1971, 2079, 2020, 2061] payload_rows_per_dst=[4194, 4020, 4192, 4238, 3808, 4158, 4126, 3960] dup_rows_per_dst=[22, 4, 0, 26, 0, 0, 20, 0] combine_rows_per_src=[4194, 4020, 4192, 4238, 3808, 4158, 4126, 3960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=236093440 dispatch_meta_bytes=115280 combine_payload_bytes=236093440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236208720 ingress_bytes=1645379584 link_bytes=1645379584 token_bytes=472302160 weight_movement_bytes=1409286144 total_nvlink_bytes=1881588304 weight_to_token_ratio=2.983866 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213438855e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233160704 dispatch_meta_bytes=113920 combine_payload_bytes=233160704 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233274624 ingress_bytes=2347089920 link_bytes=2347089920 token_bytes=466435328 weight_movement_bytes=2113929216 total_nvlink_bytes=2580364544 weight_to_token_ratio=4.532095 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314272383e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2046, 2036, 2070, 2097, 1997, 2081, 1966] payload_rows_per_dst=[4182, 4092, 4072, 4140, 4194, 3994, 4162, 3932] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4092, 4072, 4140, 4194, 3994, 4162, 3932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2122, 2054, 1980, 2062, 2051, 2101, 1965, 2049] payload_rows_per_dst=[4244, 4108, 3960, 4124, 4102, 4202, 3930, 4098] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4244, 4108, 3960, 4124, 4102, 4202, 3930, 4098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28624 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1990, 2072, 2050, 2029, 2044, 2069, 2069, 2061] payload_rows_per_dst=[3980, 4144, 4100, 4058, 4088, 4138, 4138, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3980, 4144, 4100, 4058, 4088, 4138, 4138, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2088, 2037, 2032, 2056, 2003, 2039, 2024, 2105] payload_rows_per_dst=[4176, 4074, 4064, 4112, 4006, 4078, 4048, 4210] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4176, 4074, 4064, 4112, 4006, 4078, 4048, 4210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233488384 dispatch_meta_bytes=114208 combine_payload_bytes=233488384 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233602592 ingress_bytes=2347417600 link_bytes=2347417600 token_bytes=467090976 weight_movement_bytes=2113929216 total_nvlink_bytes=2581020192 weight_to_token_ratio=4.525733 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314455870e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2028, 2087, 2042, 1981, 2016, 2057, 2117, 2056] payload_rows_per_dst=[4056, 4174, 4084, 3962, 4032, 4114, 4234, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4056, 4174, 4084, 3962, 4032, 4114, 4234, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114424 combine_payload_bytes=234340352 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234454776 ingress_bytes=3052912640 link_bytes=3052912640 token_bytes=468795128 weight_movement_bytes=2818572288 total_nvlink_bytes=3287367416 weight_to_token_ratio=6.012375 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709503644e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234126936 ingress_bytes=3757228032 link_bytes=3757228032 token_bytes=468139608 weight_movement_bytes=3523215360 total_nvlink_bytes=3991354968 weight_to_token_ratio=7.525993 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103890864e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235618304 dispatch_meta_bytes=115048 combine_payload_bytes=235618304 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235733352 ingress_bytes=3054190592 link_bytes=3054190592 token_bytes=471351656 weight_movement_bytes=2818572288 total_nvlink_bytes=3289923944 weight_to_token_ratio=5.979765 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710219243e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234487808 dispatch_meta_bytes=114496 combine_payload_bytes=234487808 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234602304 ingress_bytes=3053060096 link_bytes=3053060096 token_bytes=469090112 weight_movement_bytes=2818572288 total_nvlink_bytes=3287662400 weight_to_token_ratio=6.008595 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709586213e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2033, 2103, 1981, 2057, 2087, 2055, 2035, 2033] payload_rows_per_dst=[4066, 4206, 3962, 4114, 4174, 4110, 4070, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4066, 4206, 3962, 4114, 4174, 4110, 4070, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=28586 dup_rows=64 dup_rows_remote=56 dedup_ratio=0.001953 router_tokens_per_expert=[2083, 2058, 2040, 2040, 2076, 2056, 1991, 2040] payload_rows_per_dst=[3912, 4038, 4058, 4144, 4334, 4064, 4036, 4118] dup_rows_per_dst=[0, 0, 12, 4, 34, 0, 6, 8] combine_rows_per_src=[3912, 4038, 4058, 4144, 4334, 4064, 4036, 4118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114736 combine_payload_bytes=234979328 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235094064 ingress_bytes=3053551616 link_bytes=3053551616 token_bytes=470073392 weight_movement_bytes=2818572288 total_nvlink_bytes=3288645680 weight_to_token_ratio=5.996026 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709861443e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32714 payload_rows_remote=28556 dup_rows=54 dup_rows_remote=52 dedup_ratio=0.001648 router_tokens_per_expert=[2078, 2016, 2027, 2049, 2012, 2103, 2024, 2075] payload_rows_per_dst=[4158, 4146, 3960, 4118, 3998, 4206, 4146, 3982] dup_rows_per_dst=[2, 20, 4, 2, 0, 0, 26, 0] combine_rows_per_src=[4158, 4146, 3960, 4118, 3998, 4206, 4146, 3982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234848184 ingress_bytes=1644019712 link_bytes=1644019712 token_bytes=469581752 weight_movement_bytes=1409286144 total_nvlink_bytes=1878867896 weight_to_token_ratio=3.001152 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205824140e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234176512 dispatch_meta_bytes=114568 combine_payload_bytes=234176512 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234291080 ingress_bytes=3052748800 link_bytes=3052748800 token_bytes=468467592 weight_movement_bytes=2818572288 total_nvlink_bytes=3287039880 weight_to_token_ratio=6.016579 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709411900e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2082, 2028, 2030, 2046, 2070, 2061, 2021] payload_rows_per_dst=[4092, 4164, 4056, 4060, 4092, 4140, 4122, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4164, 4056, 4060, 4092, 4140, 4122, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2026, 2059, 2030, 2110, 2055, 2018, 2040] payload_rows_per_dst=[4092, 4052, 4118, 4060, 4220, 4110, 4036, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4052, 4118, 4060, 4220, 4110, 4036, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114432 combine_payload_bytes=233930752 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234045184 ingress_bytes=4461789184 link_bytes=4461789184 token_bytes=467975936 weight_movement_bytes=4227858432 total_nvlink_bytes=4695834368 weight_to_token_ratio=9.034350 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498415699e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28608 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2063, 2077, 2026, 2034, 2080, 2042, 2015, 2047] payload_rows_per_dst=[4126, 4154, 4052, 4068, 4160, 4084, 4030, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4126, 4154, 4052, 4068, 4160, 4084, 4030, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2050, 2096, 2085, 1966, 2026, 1994, 2095, 2072] payload_rows_per_dst=[4100, 4192, 4170, 3932, 4052, 3988, 4190, 4144] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4100, 4192, 4170, 3932, 4052, 3988, 4190, 4144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2009, 2068, 2039, 2024, 2060, 2057, 2070] payload_rows_per_dst=[4114, 4018, 4136, 4078, 4048, 4120, 4114, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4018, 4136, 4078, 4048, 4120, 4114, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234880968 ingress_bytes=2348695552 link_bytes=2348695552 token_bytes=469647304 weight_movement_bytes=2113929216 total_nvlink_bytes=2583576520 weight_to_token_ratio=4.501099 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315171470e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234668032 dispatch_meta_bytes=114584 combine_payload_bytes=234668032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234782616 ingress_bytes=1643954176 link_bytes=1643954176 token_bytes=469450648 weight_movement_bytes=1409286144 total_nvlink_bytes=1878736792 weight_to_token_ratio=3.001990 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205457165e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2038, 2092, 2044, 2051, 2005, 2066, 2031] payload_rows_per_dst=[4114, 4076, 4184, 4088, 4102, 4010, 4132, 4062] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4076, 4184, 4088, 4102, 4010, 4132, 4062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234356736 dispatch_meta_bytes=114432 combine_payload_bytes=234356736 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234471168 ingress_bytes=2348285952 link_bytes=2348285952 token_bytes=468827904 weight_movement_bytes=2113929216 total_nvlink_bytes=2582757120 weight_to_token_ratio=4.508966 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314942111e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32650 payload_rows_remote=28374 dup_rows=118 dup_rows_remote=80 dedup_ratio=0.003601 router_tokens_per_expert=[2045, 2068, 2059, 2013, 2032, 2062, 2046, 2059] payload_rows_per_dst=[3488, 4304, 4118, 4072, 4076, 4118, 4198, 4276] dup_rows_per_dst=[0, 22, 0, 36, 0, 4, 18, 38] combine_rows_per_src=[3488, 4304, 4118, 4072, 4076, 4118, 4198, 4276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234094592 dispatch_meta_bytes=114304 combine_payload_bytes=234094592 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234208896 ingress_bytes=3757309952 link_bytes=3757309952 token_bytes=468303488 weight_movement_bytes=3523215360 total_nvlink_bytes=3991518848 weight_to_token_ratio=7.523359 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103936735e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234667872 ingress_bytes=3757768704 link_bytes=3757768704 token_bytes=469221216 weight_movement_bytes=3523215360 total_nvlink_bytes=3992436576 weight_to_token_ratio=7.508645 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104193617e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32700 payload_rows_remote=28436 dup_rows=68 dup_rows_remote=58 dedup_ratio=0.002075 router_tokens_per_expert=[2085, 2071, 2037, 2019, 2017, 2066, 2063, 2026] payload_rows_per_dst=[4264, 4310, 4132, 4154, 3882, 3914, 4096, 3948] dup_rows_per_dst=[10, 38, 10, 10, 0, 0, 0, 0] combine_rows_per_src=[4264, 4310, 4132, 4154, 3882, 3914, 4096, 3948] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235061280 ingress_bytes=2348875776 link_bytes=2348875776 token_bytes=470007840 weight_movement_bytes=2113929216 total_nvlink_bytes=2583937056 weight_to_token_ratio=4.497647 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315272388e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=232439808 dispatch_meta_bytes=113816 combine_payload_bytes=232439808 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232553624 ingress_bytes=3051012096 link_bytes=3051012096 token_bytes=464993432 weight_movement_bytes=2818572288 total_nvlink_bytes=3283565720 weight_to_token_ratio=6.061531 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708439418e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=232947712 dispatch_meta_bytes=113976 combine_payload_bytes=232947712 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233061688 ingress_bytes=2346876928 link_bytes=2346876928 token_bytes=466009400 weight_movement_bytes=2113929216 total_nvlink_bytes=2579938616 weight_to_token_ratio=4.536237 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314153116e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2153, 1960, 2078, 2050, 2054, 2090, 1997, 2002] payload_rows_per_dst=[4306, 3920, 4156, 4100, 4108, 4180, 3994, 4004] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4306, 3920, 4156, 4100, 4108, 4180, 3994, 4004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2040, 2049, 2069, 1992, 2068, 2014, 2081] payload_rows_per_dst=[4142, 4080, 4098, 4138, 3984, 4136, 4028, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4080, 4098, 4138, 3984, 4136, 4028, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28674 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2095, 2004, 1969, 2042, 2047, 2078, 2101, 2048] payload_rows_per_dst=[4190, 4008, 3938, 4084, 4094, 4156, 4202, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4190, 4008, 3938, 4084, 4094, 4156, 4202, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28772 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2104, 1998, 2088, 2040, 2068, 1998, 2041, 2047] payload_rows_per_dst=[4208, 3996, 4176, 4080, 4136, 3996, 4082, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4208, 3996, 4176, 4080, 4136, 3996, 4082, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28748 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2112, 2025, 2010, 2033, 2102, 2017, 2057, 2028] payload_rows_per_dst=[4224, 4050, 4020, 4066, 4204, 4034, 4114, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4224, 4050, 4020, 4066, 4204, 4034, 4114, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234897408 dispatch_meta_bytes=114696 combine_payload_bytes=234897408 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235012104 ingress_bytes=3053469696 link_bytes=3053469696 token_bytes=469909512 weight_movement_bytes=2818572288 total_nvlink_bytes=3288481800 weight_to_token_ratio=5.998117 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709815572e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234192896 dispatch_meta_bytes=114352 combine_payload_bytes=234192896 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234307248 ingress_bytes=1643479040 link_bytes=1643479040 token_bytes=468500144 weight_movement_bytes=1409286144 total_nvlink_bytes=1877786288 weight_to_token_ratio=3.008080 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202796602e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28472 dup_rows=62 dup_rows_remote=44 dedup_ratio=0.001892 router_tokens_per_expert=[2051, 2080, 2019, 2075, 2030, 2016, 2059, 2054] payload_rows_per_dst=[4032, 4160, 3890, 4134, 4188, 4034, 4034, 4234] dup_rows_per_dst=[0, 0, 4, 0, 38, 2, 0, 18] combine_rows_per_src=[4032, 4160, 3890, 4134, 4188, 4034, 4034, 4234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114960 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235553040 ingress_bytes=1644724224 link_bytes=1644724224 token_bytes=470991120 weight_movement_bytes=1409286144 total_nvlink_bytes=1880277264 weight_to_token_ratio=2.992171 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209769113e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235700224 dispatch_meta_bytes=115088 combine_payload_bytes=235700224 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235815312 ingress_bytes=1644986368 link_bytes=1644986368 token_bytes=471515536 weight_movement_bytes=1409286144 total_nvlink_bytes=1880801680 weight_to_token_ratio=2.988843 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211237010e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32740 payload_rows_remote=28570 dup_rows=28 dup_rows_remote=24 dedup_ratio=0.000854 router_tokens_per_expert=[2076, 2020, 2149, 2016, 2063, 2019, 2023, 2018] payload_rows_per_dst=[4170, 4040, 4344, 4032, 4134, 4038, 3946, 4036] dup_rows_per_dst=[4, 0, 8, 0, 16, 0, 0, 0] combine_rows_per_src=[4170, 4040, 4344, 4032, 4134, 4038, 3946, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235503616 dispatch_meta_bytes=114992 combine_payload_bytes=235503616 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235618608 ingress_bytes=1644789760 link_bytes=1644789760 token_bytes=471122224 weight_movement_bytes=1409286144 total_nvlink_bytes=1880408368 weight_to_token_ratio=2.991339 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210136087e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233242624 dispatch_meta_bytes=114064 combine_payload_bytes=233242624 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233356688 ingress_bytes=2347171840 link_bytes=2347171840 token_bytes=466599312 weight_movement_bytes=2113929216 total_nvlink_bytes=2580528528 weight_to_token_ratio=4.530502 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314318255e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234045440 dispatch_meta_bytes=114376 combine_payload_bytes=234045440 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234159816 ingress_bytes=4461903872 link_bytes=4461903872 token_bytes=468205256 weight_movement_bytes=4227858432 total_nvlink_bytes=4696063688 weight_to_token_ratio=9.029925 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498479919e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2097, 2036, 2084, 1993, 2040, 2006, 2049, 2079] payload_rows_per_dst=[4194, 4072, 4168, 3986, 4080, 4012, 4098, 4158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4072, 4168, 3986, 4080, 4012, 4098, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2024, 2064, 2071, 2041, 2044, 2037, 2062] payload_rows_per_dst=[4082, 4048, 4128, 4142, 4082, 4088, 4074, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4048, 4128, 4142, 4082, 4088, 4074, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28542 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2006, 2030, 2082, 2018, 1998, 2051, 2113, 2086] payload_rows_per_dst=[4012, 4060, 4164, 4036, 3996, 4102, 4226, 4172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4012, 4060, 4164, 4036, 3996, 4102, 4226, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28868 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 1950, 2067, 2081, 2067, 2103, 2006, 2039] payload_rows_per_dst=[4142, 3900, 4134, 4162, 4134, 4206, 4012, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 3900, 4134, 4162, 4134, 4206, 4012, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235126848 ingress_bytes=3053584384 link_bytes=3053584384 token_bytes=470138944 weight_movement_bytes=2818572288 total_nvlink_bytes=3288711232 weight_to_token_ratio=5.995190 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709879792e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2010, 2052, 2019, 2024, 2107, 2038, 2051, 2083] payload_rows_per_dst=[4020, 4104, 4038, 4048, 4214, 4076, 4102, 4166] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4020, 4104, 4038, 4048, 4214, 4076, 4102, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32546 payload_rows_remote=28422 dup_rows=222 dup_rows_remote=208 dedup_ratio=0.006775 router_tokens_per_expert=[2152, 2011, 2103, 1952, 2065, 2073, 2018, 2010] payload_rows_per_dst=[3826, 3850, 3650, 4504, 4074, 4492, 4026, 4124] dup_rows_per_dst=[0, 6, 0, 112, 0, 82, 8, 14] combine_rows_per_src=[3826, 3850, 3650, 4504, 4074, 4492, 4026, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235061280 ingress_bytes=3758161920 link_bytes=3758161920 token_bytes=470007840 weight_movement_bytes=3523215360 total_nvlink_bytes=3993223200 weight_to_token_ratio=7.496078 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104413802e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114168 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233930232 ingress_bytes=3757031424 link_bytes=3757031424 token_bytes=467746296 weight_movement_bytes=3523215360 total_nvlink_bytes=3990961656 weight_to_token_ratio=7.532321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103780771e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=236486656 dispatch_meta_bytes=115472 combine_payload_bytes=236486656 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236602128 ingress_bytes=1645772800 link_bytes=1645772800 token_bytes=473088784 weight_movement_bytes=1409286144 total_nvlink_bytes=1882374928 weight_to_token_ratio=2.978904 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.215640701e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=28374 dup_rows=98 dup_rows_remote=70 dedup_ratio=0.002991 router_tokens_per_expert=[2092, 2000, 1997, 2024, 2087, 1982, 2080, 2122] payload_rows_per_dst=[4296, 4048, 4266, 4204, 3888, 4006, 3856, 4106] dup_rows_per_dst=[28, 12, 26, 30, 0, 2, 0, 0] combine_rows_per_src=[4296, 4048, 4266, 4204, 3888, 4006, 3856, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=114920 combine_payload_bytes=235356160 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235471080 ingress_bytes=2349285376 link_bytes=2349285376 token_bytes=470827240 weight_movement_bytes=2113929216 total_nvlink_bytes=2584756456 weight_to_token_ratio=4.489819 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315501747e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=232833024 dispatch_meta_bytes=114520 combine_payload_bytes=232833024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232947544 ingress_bytes=1642119168 link_bytes=1642119168 token_bytes=465780568 weight_movement_bytes=1409286144 total_nvlink_bytes=1875066712 weight_to_token_ratio=3.025644 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195181887e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28532 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2000, 2016, 2020, 2062, 2118, 2094, 2012, 2062] payload_rows_per_dst=[4000, 4032, 4040, 4124, 4236, 4188, 4024, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4000, 4032, 4040, 4124, 4236, 4188, 4024, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232439808 dispatch_meta_bytes=113776 combine_payload_bytes=232439808 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232553584 ingress_bytes=3051012096 link_bytes=3051012096 token_bytes=464993392 weight_movement_bytes=2818572288 total_nvlink_bytes=3283565680 weight_to_token_ratio=6.061532 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708439418e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28816 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2133, 2072, 2053, 2080, 1976, 2051, 2003] payload_rows_per_dst=[4032, 4266, 4144, 4106, 4160, 3952, 4102, 4006] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4266, 4144, 4106, 4160, 3952, 4102, 4006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2070, 2096, 2055, 1986, 2044, 1994, 2059, 2080] payload_rows_per_dst=[4140, 4192, 4110, 3972, 4088, 3988, 4118, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4192, 4110, 3972, 4088, 3988, 4118, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28522 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2073, 2123, 2010, 2033, 2040, 2025, 2097, 1983] payload_rows_per_dst=[4146, 4246, 4020, 4066, 4080, 4050, 4194, 3966] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 4246, 4020, 4066, 4080, 4050, 4194, 3966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28534 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2068, 2050, 2117, 2009, 1997, 1994, 2087, 2062] payload_rows_per_dst=[4136, 4100, 4234, 4018, 3994, 3988, 4174, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4136, 4100, 4234, 4018, 3994, 3988, 4174, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233734144 dispatch_meta_bytes=114128 combine_payload_bytes=233734144 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233848272 ingress_bytes=3756949504 link_bytes=3756949504 token_bytes=467582416 weight_movement_bytes=3523215360 total_nvlink_bytes=3990797776 weight_to_token_ratio=7.534961 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103734899e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32726 payload_rows_remote=28532 dup_rows=42 dup_rows_remote=2 dedup_ratio=0.001282 router_tokens_per_expert=[2072, 2012, 2039, 2018, 2039, 2099, 2111, 1994] payload_rows_per_dst=[3918, 4006, 4078, 4018, 4092, 4214, 4206, 4194] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 40] combine_rows_per_src=[3918, 4006, 4078, 4018, 4092, 4214, 4206, 4194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=236060672 dispatch_meta_bytes=115264 combine_payload_bytes=236060672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236175936 ingress_bytes=1645346816 link_bytes=1645346816 token_bytes=472236608 weight_movement_bytes=1409286144 total_nvlink_bytes=1881522752 weight_to_token_ratio=2.984280 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213255368e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234700800 dispatch_meta_bytes=114600 combine_payload_bytes=234700800 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234815400 ingress_bytes=3053273088 link_bytes=3053273088 token_bytes=469516200 weight_movement_bytes=2818572288 total_nvlink_bytes=3288088488 weight_to_token_ratio=6.003142 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709705479e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32756 payload_rows_remote=28660 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000366 router_tokens_per_expert=[2047, 2052, 2083, 2021, 2058, 2027, 2072, 2024] payload_rows_per_dst=[4096, 4104, 4230, 4042, 4116, 4054, 4066, 4048] dup_rows_per_dst=[0, 0, 12, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 4104, 4230, 4042, 4116, 4054, 4066, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233652224 dispatch_meta_bytes=114088 combine_payload_bytes=233652224 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233766312 ingress_bytes=3052224512 link_bytes=3052224512 token_bytes=467418536 weight_movement_bytes=2818572288 total_nvlink_bytes=3285990824 weight_to_token_ratio=6.030082 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709118321e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233750528 dispatch_meta_bytes=114136 combine_payload_bytes=233750528 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233864664 ingress_bytes=3052322816 link_bytes=3052322816 token_bytes=467615192 weight_movement_bytes=2818572288 total_nvlink_bytes=3286187480 weight_to_token_ratio=6.027546 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709173367e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233734144 dispatch_meta_bytes=114136 combine_payload_bytes=233734144 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233848280 ingress_bytes=2347663360 link_bytes=2347663360 token_bytes=467582424 weight_movement_bytes=2113929216 total_nvlink_bytes=2581511640 weight_to_token_ratio=4.520977 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314593485e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114688 combine_payload_bytes=234782720 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234897408 ingress_bytes=3053355008 link_bytes=3053355008 token_bytes=469680128 weight_movement_bytes=2818572288 total_nvlink_bytes=3288252416 weight_to_token_ratio=6.001046 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709751351e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2073, 2056, 2062, 2088, 2018, 2049, 2024, 2014] payload_rows_per_dst=[4146, 4112, 4124, 4176, 4036, 4098, 4048, 4028] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 4112, 4124, 4176, 4036, 4098, 4048, 4028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2085, 2131, 2027, 1987, 2055, 2046, 2030, 2023] payload_rows_per_dst=[4170, 4262, 4054, 3974, 4110, 4092, 4060, 4046] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4170, 4262, 4054, 3974, 4110, 4092, 4060, 4046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28626 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2040, 1969, 2019, 2053, 2085, 2035, 2071, 2112] payload_rows_per_dst=[4080, 3938, 4038, 4106, 4170, 4070, 4142, 4224] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4080, 3938, 4038, 4106, 4170, 4070, 4142, 4224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2062, 2096, 2005, 2112, 2070, 1961, 2056, 2022] payload_rows_per_dst=[4124, 4192, 4010, 4224, 4140, 3922, 4112, 4044] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4124, 4192, 4010, 4224, 4140, 3922, 4112, 4044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235487472 ingress_bytes=1644658688 link_bytes=1644658688 token_bytes=470860016 weight_movement_bytes=1409286144 total_nvlink_bytes=1880146160 weight_to_token_ratio=2.993004 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209402138e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 2068, 2077, 1991, 2043, 2070, 2120, 1984] payload_rows_per_dst=[4062, 4136, 4154, 3982, 4086, 4140, 4240, 3968] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 4136, 4154, 3982, 4086, 4140, 4240, 3968] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234913792 dispatch_meta_bytes=114704 combine_payload_bytes=234913792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235028496 ingress_bytes=1644199936 link_bytes=1644199936 token_bytes=469942288 weight_movement_bytes=1409286144 total_nvlink_bytes=1879228432 weight_to_token_ratio=2.998849 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206833319e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234504192 dispatch_meta_bytes=114504 combine_payload_bytes=234504192 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234618696 ingress_bytes=3757719552 link_bytes=3757719552 token_bytes=469122888 weight_movement_bytes=3523215360 total_nvlink_bytes=3992338248 weight_to_token_ratio=7.510218 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104166094e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32626 payload_rows_remote=28202 dup_rows=142 dup_rows_remote=82 dedup_ratio=0.004333 router_tokens_per_expert=[2057, 2038, 2061, 2096, 1992, 2074, 2027, 2039] payload_rows_per_dst=[3746, 4176, 3800, 4186, 4194, 4260, 3840, 4424] dup_rows_per_dst=[0, 16, 0, 8, 38, 20, 0, 60] combine_rows_per_src=[3746, 4176, 3800, 4186, 4194, 4260, 3840, 4424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32732 payload_rows_remote=28606 dup_rows=36 dup_rows_remote=20 dedup_ratio=0.001099 router_tokens_per_expert=[2017, 2076, 2026, 2067, 2053, 2017, 2106, 2022] payload_rows_per_dst=[4126, 4220, 4148, 4166, 3902, 3936, 4210, 4024] dup_rows_per_dst=[16, 6, 6, 8, 0, 0, 0, 0] combine_rows_per_src=[4126, 4220, 4148, 4166, 3902, 3936, 4210, 4024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234094592 dispatch_meta_bytes=114304 combine_payload_bytes=234094592 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234208896 ingress_bytes=2348023808 link_bytes=2348023808 token_bytes=468303488 weight_movement_bytes=2113929216 total_nvlink_bytes=2582232704 weight_to_token_ratio=4.514016 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314795321e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28770 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2008, 2070, 2165, 1999, 2070, 2095, 2011, 1966] payload_rows_per_dst=[4016, 4140, 4330, 3998, 4140, 4190, 4022, 3932] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4016, 4140, 4330, 3998, 4140, 4190, 4022, 3932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234520344 ingress_bytes=1643692032 link_bytes=1643692032 token_bytes=468926232 weight_movement_bytes=1409286144 total_nvlink_bytes=1878212376 weight_to_token_ratio=3.005347 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203989268e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=231030784 dispatch_meta_bytes=113136 combine_payload_bytes=231030784 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231143920 ingress_bytes=1640316928 link_bytes=1640316928 token_bytes=462174704 weight_movement_bytes=1409286144 total_nvlink_bytes=1871460848 weight_to_token_ratio=3.049250 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.185090095e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114504 combine_payload_bytes=234340352 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234454856 ingress_bytes=1643626496 link_bytes=1643626496 token_bytes=468795208 weight_movement_bytes=1409286144 total_nvlink_bytes=1878081352 weight_to_token_ratio=3.006187 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203622294e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2081, 2111, 2028, 2041, 2058, 1973, 1998, 2094] payload_rows_per_dst=[4162, 4222, 4056, 4082, 4116, 3946, 3996, 4188] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4162, 4222, 4056, 4082, 4116, 3946, 3996, 4188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2022, 2069, 2079, 2007, 2009, 2093, 2049] payload_rows_per_dst=[4112, 4044, 4138, 4158, 4014, 4018, 4186, 4098] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4044, 4138, 4158, 4014, 4018, 4186, 4098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28558 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2121, 2054, 2006, 1991, 2114, 2050, 2105, 1943] payload_rows_per_dst=[4242, 4108, 4012, 3982, 4228, 4100, 4210, 3886] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4242, 4108, 4012, 3982, 4228, 4100, 4210, 3886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235683840 dispatch_meta_bytes=115080 combine_payload_bytes=235683840 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235798920 ingress_bytes=2349613056 link_bytes=2349613056 token_bytes=471482760 weight_movement_bytes=2113929216 total_nvlink_bytes=2585411976 weight_to_token_ratio=4.483577 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315685234e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234831792 ingress_bytes=4462575616 link_bytes=4462575616 token_bytes=469548976 weight_movement_bytes=4227858432 total_nvlink_bytes=4697407408 weight_to_token_ratio=9.004084 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498856068e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28872 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 1948, 2091, 2011, 2142, 2035, 2116, 1982] payload_rows_per_dst=[4118, 3896, 4182, 4022, 4284, 4070, 4232, 3964] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 3896, 4182, 4022, 4284, 4070, 4232, 3964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2105, 2026, 2071, 1947, 2053, 1996, 2125] payload_rows_per_dst=[4122, 4210, 4052, 4142, 3894, 4106, 3992, 4250] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4210, 4052, 4142, 3894, 4106, 3992, 4250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32732 payload_rows_remote=28662 dup_rows=36 dup_rows_remote=30 dedup_ratio=0.001099 router_tokens_per_expert=[2022, 1996, 2071, 2081, 2047, 2078, 2074, 2015] payload_rows_per_dst=[4044, 3992, 3968, 4158, 4028, 4262, 4210, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 26, 4, 6] combine_rows_per_src=[4044, 3992, 3968, 4158, 4028, 4262, 4210, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=235635000 ingress_bytes=940163072 link_bytes=940163072 token_bytes=471155000 weight_movement_bytes=704643072 total_nvlink_bytes=1175798072 weight_to_token_ratio=1.495565 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.264520760e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233947136 dispatch_meta_bytes=114232 combine_payload_bytes=233947136 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234061368 ingress_bytes=3757162496 link_bytes=3757162496 token_bytes=468008504 weight_movement_bytes=3523215360 total_nvlink_bytes=3991223864 weight_to_token_ratio=7.528101 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103854166e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32610 payload_rows_remote=28280 dup_rows=158 dup_rows_remote=124 dedup_ratio=0.004822 router_tokens_per_expert=[2080, 2005, 2040, 1993, 1964, 2119, 2081, 2102] payload_rows_per_dst=[4330, 4022, 4282, 4124, 4324, 3944, 3748, 3836] dup_rows_per_dst=[34, 4, 30, 32, 56, 2, 0, 0] combine_rows_per_src=[4330, 4022, 4282, 4124, 4324, 3944, 3748, 3836] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=236519424 dispatch_meta_bytes=115488 combine_payload_bytes=236519424 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=236634912 ingress_bytes=1645805568 link_bytes=1645805568 token_bytes=473154336 weight_movement_bytes=1409286144 total_nvlink_bytes=1882440480 weight_to_token_ratio=2.978491 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.215824188e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28796 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 1998, 2045, 1986, 2071, 2083, 2051, 2090] payload_rows_per_dst=[4120, 3996, 4090, 3972, 4142, 4166, 4102, 4180] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 3996, 4090, 3972, 4142, 4166, 4102, 4180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235356336 ingress_bytes=3758456832 link_bytes=3758456832 token_bytes=470597808 weight_movement_bytes=3523215360 total_nvlink_bytes=3993813168 weight_to_token_ratio=7.486680 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104578940e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114768 combine_payload_bytes=234799104 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234913872 ingress_bytes=3053371392 link_bytes=3053371392 token_bytes=469712976 weight_movement_bytes=2818572288 total_nvlink_bytes=3288285264 weight_to_token_ratio=6.000627 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709760526e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28630 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2017, 2024, 2055, 2051, 2069, 2060, 2058, 2050] payload_rows_per_dst=[4034, 4048, 4110, 4102, 4138, 4120, 4116, 4100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4034, 4048, 4110, 4102, 4138, 4120, 4116, 4100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=231669760 dispatch_meta_bytes=113616 combine_payload_bytes=231669760 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231783376 ingress_bytes=2345598976 link_bytes=2345598976 token_bytes=463453136 weight_movement_bytes=2113929216 total_nvlink_bytes=2577382352 weight_to_token_ratio=4.561258 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313437516e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2114, 2089, 2059, 2071, 2072, 1977, 2040, 1962] payload_rows_per_dst=[4228, 4178, 4118, 4142, 4144, 3954, 4080, 3924] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4228, 4178, 4118, 4142, 4144, 3954, 4080, 3924] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235896832 dispatch_meta_bytes=115184 combine_payload_bytes=235896832 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236012016 ingress_bytes=1645182976 link_bytes=1645182976 token_bytes=471908848 weight_movement_bytes=1409286144 total_nvlink_bytes=1881194992 weight_to_token_ratio=2.986352 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212337932e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1966, 2016, 2061, 2024, 2061, 2122, 2048, 2086] payload_rows_per_dst=[3932, 4032, 4122, 4048, 4122, 4244, 4096, 4172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3932, 4032, 4122, 4048, 4122, 4244, 4096, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114520 combine_payload_bytes=234536960 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234651480 ingress_bytes=1643823104 link_bytes=1643823104 token_bytes=469188440 weight_movement_bytes=1409286144 total_nvlink_bytes=1878474584 weight_to_token_ratio=3.003668 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204723217e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1995, 2037, 2036, 2032, 2052, 2105, 2060, 2067] payload_rows_per_dst=[3990, 4074, 4072, 4064, 4104, 4210, 4120, 4134] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3990, 4074, 4072, 4064, 4104, 4210, 4120, 4134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7464410016 prefetch_bytes=19025362944 reduce_bytes=17616076800 weight_movement_bytes=36641439744 egress_bytes=3733117856 ingress_bytes=40372731904 link_bytes=40372731904 total_nvlink_bytes=44105849760 weight_to_token_ratio=4.908819 prefetch_share_of_ingress=0.471243 dedup_ratio=0.002354 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=25 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.412940289e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=236044288 dispatch_meta_bytes=115256 combine_payload_bytes=236044288 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=236159544 ingress_bytes=3759259648 link_bytes=3759259648 token_bytes=472203832 weight_movement_bytes=3523215360 total_nvlink_bytes=3995419192 weight_to_token_ratio=7.461217 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.105028484e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2124, 1969, 2048, 2007, 2082, 2052, 2020, 2082] payload_rows_per_dst=[4248, 3938, 4096, 4014, 4164, 4104, 4040, 4164] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4248, 3938, 4096, 4014, 4164, 4104, 4040, 4164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32730 payload_rows_remote=28500 dup_rows=38 dup_rows_remote=34 dedup_ratio=0.001160 router_tokens_per_expert=[2001, 2028, 2010, 2121, 1996, 2005, 2115, 2108] payload_rows_per_dst=[3888, 4144, 4020, 4242, 3992, 3980, 4234, 4230] dup_rows_per_dst=[0, 26, 0, 0, 0, 0, 8, 4] combine_rows_per_src=[3888, 4144, 4020, 4242, 3992, 3980, 4234, 4230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234995712 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=469876736 weight_movement_bytes=1409286144 total_nvlink_bytes=1879162880 weight_to_token_ratio=2.999268 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7519568248 prefetch_bytes=16911433728 reduce_bytes=21139292160 weight_movement_bytes=38050725888 egress_bytes=3760701816 ingress_bytes=41809592320 link_bytes=41809592320 total_nvlink_bytes=45570294136 weight_to_token_ratio=5.060227 prefetch_share_of_ingress=0.404487 dedup_ratio=0.000000 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=30 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.463226655e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235176024 ingress_bytes=1644347392 link_bytes=1644347392 token_bytes=470237272 weight_movement_bytes=1409286144 total_nvlink_bytes=1879523416 weight_to_token_ratio=2.996968 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207659011e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2071, 2051, 2053, 2023, 2066, 2064, 2011] payload_rows_per_dst=[4090, 4142, 4102, 4106, 4046, 4132, 4128, 4022] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4142, 4102, 4106, 4046, 4132, 4128, 4022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234995712 ingress_bytes=2348810240 link_bytes=2348810240 token_bytes=469876736 weight_movement_bytes=2113929216 total_nvlink_bytes=2583805952 weight_to_token_ratio=4.498902 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315235690e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114136 combine_payload_bytes=233472000 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233586136 ingress_bytes=3756687360 link_bytes=3756687360 token_bytes=467058136 weight_movement_bytes=3523215360 total_nvlink_bytes=3990273496 weight_to_token_ratio=7.543419 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103588110e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234913752 ingress_bytes=1644085248 link_bytes=1644085248 token_bytes=469712856 weight_movement_bytes=1409286144 total_nvlink_bytes=1878999000 weight_to_token_ratio=3.000314 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206191114e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2096, 2021, 2004, 2029, 2061, 2067, 2085] payload_rows_per_dst=[4042, 4192, 4042, 4008, 4058, 4122, 4134, 4170] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4192, 4042, 4008, 4058, 4122, 4134, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2054, 2044, 2029, 1989, 2113, 2104, 2035] payload_rows_per_dst=[4032, 4108, 4088, 4058, 3978, 4226, 4208, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4108, 4088, 4058, 3978, 4226, 4208, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7466671352 prefetch_bytes=19730006016 reduce_bytes=20434649088 weight_movement_bytes=40164655104 egress_bytes=3734248696 ingress_bytes=43897077760 link_bytes=43897077760 total_nvlink_bytes=47631326456 weight_to_token_ratio=5.379192 prefetch_share_of_ingress=0.449461 dedup_ratio=0.002468 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=29 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.536283199e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2116, 2042, 2016, 2021, 2040, 2040, 2034, 2075] payload_rows_per_dst=[4232, 4084, 4032, 4042, 4080, 4080, 4068, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4232, 4084, 4032, 4042, 4080, 4080, 4068, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234668032 dispatch_meta_bytes=114584 combine_payload_bytes=234668032 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234782616 ingress_bytes=4462526464 link_bytes=4462526464 token_bytes=469450648 weight_movement_bytes=4227858432 total_nvlink_bytes=4697309080 weight_to_token_ratio=9.005970 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498828545e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114240 combine_payload_bytes=233963520 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234077760 ingress_bytes=4461821952 link_bytes=4461821952 token_bytes=468041280 weight_movement_bytes=4227858432 total_nvlink_bytes=4695899712 weight_to_token_ratio=9.033089 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498434048e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28776 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2033, 1996, 2137, 1959, 2040, 2067, 2096] payload_rows_per_dst=[4112, 4066, 3992, 4274, 3918, 4080, 4134, 4192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4066, 3992, 4274, 3918, 4080, 4134, 4192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2093, 2051, 2109, 2026, 1982, 2021, 2097, 2005] payload_rows_per_dst=[4186, 4102, 4218, 4052, 3964, 4042, 4194, 4010] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4186, 4102, 4218, 4052, 3964, 4042, 4194, 4010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7519961560 prefetch_bytes=17616076800 reduce_bytes=19730006016 weight_movement_bytes=37346082816 egress_bytes=3760898520 ingress_bytes=41105145856 link_bytes=41105145856 total_nvlink_bytes=44866044376 weight_to_token_ratio=4.966260 prefetch_share_of_ingress=0.428561 dedup_ratio=0.000000 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=28 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.438572867e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7507539456 prefetch_bytes=19025362944 reduce_bytes=22548578304 weight_movement_bytes=41573941248 egress_bytes=3754685952 ingress_bytes=45326794752 link_bytes=45326794752 total_nvlink_bytes=49081480704 weight_to_token_ratio=5.537625 prefetch_share_of_ingress=0.419738 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=32 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.586319564e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235732992 dispatch_meta_bytes=115104 combine_payload_bytes=235732992 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235848096 ingress_bytes=1645019136 link_bytes=1645019136 token_bytes=471581088 weight_movement_bytes=1409286144 total_nvlink_bytes=1880867232 weight_to_token_ratio=2.988428 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211420497e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114736 combine_payload_bytes=234979328 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235094064 ingress_bytes=1644265472 link_bytes=1644265472 token_bytes=470073392 weight_movement_bytes=1409286144 total_nvlink_bytes=1879359536 weight_to_token_ratio=2.998013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207200293e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235356336 ingress_bytes=2349170688 link_bytes=2349170688 token_bytes=470597808 weight_movement_bytes=2113929216 total_nvlink_bytes=2584527024 weight_to_token_ratio=4.492008 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315437526e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7520059888 prefetch_bytes=19730006016 reduce_bytes=18320719872 weight_movement_bytes=38050725888 egress_bytes=3760947696 ingress_bytes=41809838080 link_bytes=41809838080 total_nvlink_bytes=45570785776 weight_to_token_ratio=5.059897 prefetch_share_of_ingress=0.471899 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=26 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.463235256e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7518224432 prefetch_bytes=18320719872 reduce_bytes=18320719872 weight_movement_bytes=36641439744 egress_bytes=3760029744 ingress_bytes=40399634432 link_bytes=40399634432 total_nvlink_bytes=44159664176 weight_to_token_ratio=4.873683 prefetch_share_of_ingress=0.453487 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=26 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.413881807e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28682 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2011, 2053, 2091, 2043, 2094, 2043, 2006, 2043] payload_rows_per_dst=[4022, 4106, 4182, 4086, 4188, 4086, 4012, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4022, 4106, 4182, 4086, 4188, 4086, 4012, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234962944 dispatch_meta_bytes=114728 combine_payload_bytes=234962944 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235077672 ingress_bytes=1644249088 link_bytes=1644249088 token_bytes=470040616 weight_movement_bytes=1409286144 total_nvlink_bytes=1879326760 weight_to_token_ratio=2.998222 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207108549e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28882 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2032, 2037, 2086, 1943, 2112, 2107, 2059, 2008] payload_rows_per_dst=[4064, 4074, 4172, 3886, 4224, 4214, 4118, 4016] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4064, 4074, 4172, 3886, 4224, 4214, 4118, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=236601344 dispatch_meta_bytes=115528 combine_payload_bytes=236601344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236716872 ingress_bytes=1645887488 link_bytes=1645887488 token_bytes=473318216 weight_movement_bytes=1409286144 total_nvlink_bytes=1882604360 weight_to_token_ratio=2.977460 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.216282906e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7526713416 prefetch_bytes=20434649088 reduce_bytes=12683575296 weight_movement_bytes=33118224384 egress_bytes=3764275272 ingress_bytes=36880662528 link_bytes=36880662528 total_nvlink_bytes=40644937800 weight_to_token_ratio=4.400091 prefetch_share_of_ingress=0.554075 dedup_ratio=0.000000 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=18 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.290726971e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none - [2026-09-16 21:35:52] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 578.5 | throughput per GPU (TFLOP/s/GPU): 576.3 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.270194E+01 | loss scale: 1.0 | grad norm: 39.670 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28718 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1989, 2081, 2127, 2045, 2025, 2053, 2002, 2062] payload_rows_per_dst=[3978, 4162, 4254, 4090, 4050, 4106, 4004, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3978, 4162, 4254, 4090, 4050, 4106, 4004, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32690 payload_rows_remote=28500 dup_rows=78 dup_rows_remote=72 dedup_ratio=0.002380 router_tokens_per_expert=[2067, 2039, 2025, 2064, 2038, 2050, 2027, 2074] payload_rows_per_dst=[4190, 4212, 4050, 4204, 4056, 3846, 4054, 4078] dup_rows_per_dst=[6, 30, 0, 42, 0, 0, 0, 0] combine_rows_per_src=[4190, 4212, 4050, 4204, 4056, 3846, 4054, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235257856 dispatch_meta_bytes=114872 combine_payload_bytes=235257856 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235372728 ingress_bytes=3758473216 link_bytes=3758473216 token_bytes=470630584 weight_movement_bytes=3523215360 total_nvlink_bytes=3993845944 weight_to_token_ratio=7.486159 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104588115e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28908 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2131, 2085, 1930, 2094, 2013, 2042, 2009] payload_rows_per_dst=[4160, 4262, 4170, 3860, 4188, 4026, 4084, 4018] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4262, 4170, 3860, 4188, 4026, 4084, 4018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32634 payload_rows_remote=28562 dup_rows=134 dup_rows_remote=124 dedup_ratio=0.004089 router_tokens_per_expert=[2043, 1981, 2061, 2072, 2112, 2054, 2065, 1996] payload_rows_per_dst=[3986, 3474, 4238, 4104, 4056, 4250, 4454, 4072] dup_rows_per_dst=[0, 0, 26, 0, 0, 24, 74, 10] combine_rows_per_src=[3986, 3474, 4238, 4104, 4056, 4250, 4454, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2108, 2093, 1972, 2067, 1988, 2000, 2110] payload_rows_per_dst=[4092, 4216, 4186, 3944, 4134, 3976, 4000, 4220] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4216, 4186, 3944, 4134, 3976, 4000, 4220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114288 combine_payload_bytes=233472000 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233586288 ingress_bytes=3052044288 link_bytes=3052044288 token_bytes=467058288 weight_movement_bytes=2818572288 total_nvlink_bytes=3285630576 weight_to_token_ratio=6.034733 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709017403e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1993, 2035, 2058, 2078, 2081, 2116, 2010, 2013] payload_rows_per_dst=[3986, 4070, 4116, 4156, 4162, 4232, 4020, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3986, 4070, 4116, 4156, 4162, 4232, 4020, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28878 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2125, 1945, 2105, 2046, 2008, 2016, 2081] payload_rows_per_dst=[4116, 4250, 3890, 4210, 4092, 4016, 4032, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4250, 3890, 4210, 4092, 4016, 4032, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=236814336 dispatch_meta_bytes=115632 combine_payload_bytes=236814336 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236929968 ingress_bytes=3055386624 link_bytes=3055386624 token_bytes=473744304 weight_movement_bytes=2818572288 total_nvlink_bytes=3292316592 weight_to_token_ratio=5.949564 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710888971e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233979904 dispatch_meta_bytes=114744 combine_payload_bytes=233979904 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234094648 ingress_bytes=3052552192 link_bytes=3052552192 token_bytes=468074552 weight_movement_bytes=2818572288 total_nvlink_bytes=3286646840 weight_to_token_ratio=6.021631 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709301808e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28476 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2127, 2046, 1919, 2079, 2015, 2146, 2023, 2029] payload_rows_per_dst=[4254, 4092, 3838, 4158, 4030, 4292, 4046, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4254, 4092, 3838, 4158, 4030, 4292, 4046, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2017, 2035, 2057, 2113, 2014, 2064, 2028, 2056] payload_rows_per_dst=[4034, 4070, 4114, 4226, 4028, 4128, 4056, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4034, 4070, 4114, 4226, 4028, 4128, 4056, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235667456 dispatch_meta_bytes=115072 combine_payload_bytes=235667456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235782528 ingress_bytes=1644953600 link_bytes=1644953600 token_bytes=471449984 weight_movement_bytes=1409286144 total_nvlink_bytes=1880736128 weight_to_token_ratio=2.989259 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211053523e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235208808 ingress_bytes=3053666304 link_bytes=3053666304 token_bytes=470302824 weight_movement_bytes=2818572288 total_nvlink_bytes=3288875112 weight_to_token_ratio=5.993101 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709925664e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28534 dup_rows=80 dup_rows_remote=64 dedup_ratio=0.002441 router_tokens_per_expert=[2034, 1966, 2035, 2119, 2050, 2012, 2136, 2032] payload_rows_per_dst=[4154, 4230, 4056, 4096, 4100, 4024, 4172, 3856] dup_rows_per_dst=[16, 64, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4154, 4230, 4056, 4096, 4100, 4024, 4172, 3856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=236568576 dispatch_meta_bytes=115512 combine_payload_bytes=236568576 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=236684088 ingress_bytes=1645854720 link_bytes=1645854720 token_bytes=473252664 weight_movement_bytes=1409286144 total_nvlink_bytes=1882538808 weight_to_token_ratio=2.977873 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.216099418e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114960 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235553040 ingress_bytes=2349367296 link_bytes=2349367296 token_bytes=470991120 weight_movement_bytes=2113929216 total_nvlink_bytes=2584920336 weight_to_token_ratio=4.488257 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315547618e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233275392 dispatch_meta_bytes=113904 combine_payload_bytes=233275392 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233389296 ingress_bytes=2347204608 link_bytes=2347204608 token_bytes=466664688 weight_movement_bytes=2113929216 total_nvlink_bytes=2580593904 weight_to_token_ratio=4.529868 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314336603e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233750528 dispatch_meta_bytes=114392 combine_payload_bytes=233750528 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233864920 ingress_bytes=2347679744 link_bytes=2347679744 token_bytes=467615448 weight_movement_bytes=2113929216 total_nvlink_bytes=2581544664 weight_to_token_ratio=4.520657 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314602660e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28342 dup_rows=84 dup_rows_remote=38 dedup_ratio=0.002563 router_tokens_per_expert=[2072, 2047, 1948, 2028, 2092, 2091, 2073, 2033] payload_rows_per_dst=[4132, 4036, 3804, 3840, 4184, 4256, 4090, 4342] dup_rows_per_dst=[0, 0, 0, 0, 12, 18, 8, 46] combine_rows_per_src=[4132, 4036, 3804, 3840, 4184, 4256, 4090, 4342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28528 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 1970, 2082, 2120, 2089, 2040, 2034, 1991] payload_rows_per_dst=[4116, 3940, 4164, 4240, 4178, 4080, 4068, 3982] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 3940, 4164, 4240, 4178, 4080, 4068, 3982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2014, 2076, 2030, 2077, 2015, 2077, 2077, 2018] payload_rows_per_dst=[4028, 4152, 4060, 4154, 4030, 4154, 4154, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4028, 4152, 4060, 4154, 4030, 4154, 4154, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1996, 2076, 2031, 2003, 2087, 2060, 2071, 2060] payload_rows_per_dst=[3992, 4152, 4062, 4006, 4174, 4120, 4142, 4120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3992, 4152, 4062, 4006, 4174, 4120, 4142, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28546 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2022, 2111, 2015, 2014, 2033, 2052, 2076] payload_rows_per_dst=[4122, 4044, 4222, 4030, 4028, 4066, 4104, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4044, 4222, 4030, 4028, 4066, 4104, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=232177664 dispatch_meta_bytes=113520 combine_payload_bytes=232177664 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232291184 ingress_bytes=2346106880 link_bytes=2346106880 token_bytes=464468848 weight_movement_bytes=2113929216 total_nvlink_bytes=2578398064 weight_to_token_ratio=4.551283 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313721921e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2019, 2057, 2108, 2045, 2057, 1978, 2057, 2063] payload_rows_per_dst=[4038, 4114, 4216, 4090, 4114, 3956, 4114, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4038, 4114, 4216, 4090, 4114, 3956, 4114, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233701376 dispatch_meta_bytes=114112 combine_payload_bytes=233701376 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233815488 ingress_bytes=3052273664 link_bytes=3052273664 token_bytes=467516864 weight_movement_bytes=2818572288 total_nvlink_bytes=3286089152 weight_to_token_ratio=6.028814 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709145844e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234520344 ingress_bytes=3052978176 link_bytes=3052978176 token_bytes=468926232 weight_movement_bytes=2818572288 total_nvlink_bytes=3287498520 weight_to_token_ratio=6.010694 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709540341e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 2, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234242048 dispatch_meta_bytes=114376 combine_payload_bytes=234242048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234356424 ingress_bytes=1643528192 link_bytes=1643528192 token_bytes=468598472 weight_movement_bytes=1409286144 total_nvlink_bytes=1877884616 weight_to_token_ratio=3.007449 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203071833e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=28558 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.001587 router_tokens_per_expert=[2079, 2048, 2087, 2065, 1982, 2076, 2028, 2019] payload_rows_per_dst=[4158, 4212, 4174, 4152, 4052, 4244, 3890, 3834] dup_rows_per_dst=[0, 20, 0, 4, 8, 20, 0, 0] combine_rows_per_src=[4158, 4212, 4174, 4152, 4052, 4244, 3890, 3834] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28682 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2059, 2066, 2046, 2047, 2043, 2001, 2058] payload_rows_per_dst=[4128, 4118, 4132, 4092, 4094, 4086, 4002, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4118, 4132, 4092, 4094, 4086, 4002, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233848832 dispatch_meta_bytes=114184 combine_payload_bytes=233848832 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=233963016 ingress_bytes=4461707264 link_bytes=4461707264 token_bytes=467811848 weight_movement_bytes=4227858432 total_nvlink_bytes=4695670280 weight_to_token_ratio=9.037519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498369827e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234848184 ingress_bytes=2348662784 link_bytes=2348662784 token_bytes=469581752 weight_movement_bytes=2113929216 total_nvlink_bytes=2583510968 weight_to_token_ratio=4.501728 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315153121e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233947136 dispatch_meta_bytes=114440 combine_payload_bytes=233947136 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=234061576 ingress_bytes=5166448640 link_bytes=5166448640 token_bytes=468008712 weight_movement_bytes=4932501504 total_nvlink_bytes=5400510216 weight_to_token_ratio=10.539337 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.892995580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234962944 dispatch_meta_bytes=114728 combine_payload_bytes=234962944 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235077672 ingress_bytes=1644249088 link_bytes=1644249088 token_bytes=470040616 weight_movement_bytes=1409286144 total_nvlink_bytes=1879326760 weight_to_token_ratio=2.998222 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207108549e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32700 payload_rows_remote=28490 dup_rows=68 dup_rows_remote=50 dedup_ratio=0.002075 router_tokens_per_expert=[2160, 1992, 2009, 2008, 2070, 2022, 2061, 2062] payload_rows_per_dst=[4062, 4006, 4154, 4056, 4140, 4044, 4028, 4210] dup_rows_per_dst=[0, 4, 36, 8, 0, 0, 2, 18] combine_rows_per_src=[4062, 4006, 4154, 4056, 4140, 4044, 4028, 4210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2023, 2075, 2009, 2061, 2001, 2044, 2099, 2072] payload_rows_per_dst=[4046, 4150, 4018, 4122, 4002, 4088, 4198, 4144] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4046, 4150, 4018, 4122, 4002, 4088, 4198, 4144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2093, 1987, 2076, 2106, 2035, 2046, 2019, 2022] payload_rows_per_dst=[4186, 3974, 4152, 4212, 4070, 4092, 4038, 4044] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4186, 3974, 4152, 4212, 4070, 4092, 4038, 4044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2065, 2107, 2008, 2097, 1956, 2055, 2016] payload_rows_per_dst=[4160, 4130, 4214, 4016, 4194, 3912, 4110, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4130, 4214, 4016, 4194, 3912, 4110, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28626 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 2000, 2071, 2089, 2107, 1951, 2073, 2058] payload_rows_per_dst=[4070, 4000, 4142, 4178, 4214, 3902, 4146, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 4000, 4142, 4178, 4214, 3902, 4146, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28668 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2125, 2050, 2013, 1973, 2042, 2086, 2027, 2068] payload_rows_per_dst=[4250, 4100, 4026, 3946, 4084, 4172, 4054, 4136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4250, 4100, 4026, 3946, 4084, 4172, 4054, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233390080 dispatch_meta_bytes=114160 combine_payload_bytes=233390080 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233504240 ingress_bytes=3756605440 link_bytes=3756605440 token_bytes=466894320 weight_movement_bytes=3523215360 total_nvlink_bytes=3990109680 weight_to_token_ratio=7.546066 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103542238e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234078208 dispatch_meta_bytes=114296 combine_payload_bytes=234078208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234192504 ingress_bytes=1643364352 link_bytes=1643364352 token_bytes=468270712 weight_movement_bytes=1409286144 total_nvlink_bytes=1877556856 weight_to_token_ratio=3.009554 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202154397e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234668032 dispatch_meta_bytes=114584 combine_payload_bytes=234668032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234782616 ingress_bytes=1643954176 link_bytes=1643954176 token_bytes=469450648 weight_movement_bytes=1409286144 total_nvlink_bytes=1878736792 weight_to_token_ratio=3.001990 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205457165e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=114920 combine_payload_bytes=235356160 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=235471080 ingress_bytes=4463214592 link_bytes=4463214592 token_bytes=470827240 weight_movement_bytes=4227858432 total_nvlink_bytes=4698685672 weight_to_token_ratio=8.979639 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499213868e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32738 payload_rows_remote=28518 dup_rows=30 dup_rows_remote=20 dedup_ratio=0.000916 router_tokens_per_expert=[2087, 2068, 2036, 2061, 2037, 2054, 1999, 2042] payload_rows_per_dst=[4220, 4148, 4068, 4104, 4154, 4108, 3998, 3938] dup_rows_per_dst=[10, 4, 2, 0, 14, 0, 0, 0] combine_rows_per_src=[4220, 4148, 4068, 4104, 4154, 4108, 3998, 3938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28682 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2002, 2075, 2002, 2042, 2012, 2043, 2135, 2073] payload_rows_per_dst=[4004, 4150, 4004, 4084, 4024, 4086, 4270, 4146] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4004, 4150, 4004, 4084, 4024, 4086, 4270, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234504192 dispatch_meta_bytes=114504 combine_payload_bytes=234504192 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234618696 ingress_bytes=1643790336 link_bytes=1643790336 token_bytes=469122888 weight_movement_bytes=1409286144 total_nvlink_bytes=1878409032 weight_to_token_ratio=3.004087 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204539730e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234848256 dispatch_meta_bytes=114672 combine_payload_bytes=234848256 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234962928 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=469811184 weight_movement_bytes=1409286144 total_nvlink_bytes=1879097328 weight_to_token_ratio=2.999686 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233619456 dispatch_meta_bytes=114152 combine_payload_bytes=233619456 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233733608 ingress_bytes=2347548672 link_bytes=2347548672 token_bytes=467353064 weight_movement_bytes=2113929216 total_nvlink_bytes=2581282280 weight_to_token_ratio=4.523195 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314529265e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234962944 dispatch_meta_bytes=114728 combine_payload_bytes=234962944 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235077672 ingress_bytes=1644249088 link_bytes=1644249088 token_bytes=470040616 weight_movement_bytes=1409286144 total_nvlink_bytes=1879326760 weight_to_token_ratio=2.998222 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207108549e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28536 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2015, 2031, 2100, 2048, 2116, 2025, 2037, 2012] payload_rows_per_dst=[4030, 4062, 4200, 4096, 4232, 4050, 4074, 4024] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4030, 4062, 4200, 4096, 4232, 4050, 4074, 4024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32660 payload_rows_remote=28500 dup_rows=108 dup_rows_remote=96 dedup_ratio=0.003296 router_tokens_per_expert=[2073, 2041, 2064, 2038, 2034, 2063, 2037, 2034] payload_rows_per_dst=[4080, 3734, 4100, 3782, 4292, 4382, 4130, 4160] dup_rows_per_dst=[0, 0, 0, 0, 32, 42, 22, 12] combine_rows_per_src=[4080, 3734, 4100, 3782, 4292, 4382, 4130, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2003, 2123, 1994, 2040, 2066, 2030, 2063, 2065] payload_rows_per_dst=[4006, 4246, 3988, 4080, 4132, 4060, 4126, 4130] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4006, 4246, 3988, 4080, 4132, 4060, 4126, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28746 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2085, 2030, 2014, 2091, 1973, 2114, 2011, 2066] payload_rows_per_dst=[4170, 4060, 4028, 4182, 3946, 4228, 4022, 4132] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4170, 4060, 4028, 4182, 3946, 4228, 4022, 4132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233766912 dispatch_meta_bytes=114144 combine_payload_bytes=233766912 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=233881056 ingress_bytes=5870911488 link_bytes=5870911488 token_bytes=467647968 weight_movement_bytes=5637144576 total_nvlink_bytes=6104792544 weight_to_token_ratio=12.054248 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.287465369e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114384 combine_payload_bytes=233472000 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233586384 ingress_bytes=3052044288 link_bytes=3052044288 token_bytes=467058384 weight_movement_bytes=2818572288 total_nvlink_bytes=3285630672 weight_to_token_ratio=6.034732 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709017403e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235126848 ingress_bytes=3758227456 link_bytes=3758227456 token_bytes=470138944 weight_movement_bytes=3523215360 total_nvlink_bytes=3993354304 weight_to_token_ratio=7.493987 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104450499e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1983, 2079, 2042, 2121, 2020, 1987, 2055, 2097] payload_rows_per_dst=[3966, 4158, 4084, 4242, 4040, 3974, 4110, 4194] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3966, 4158, 4084, 4242, 4040, 3974, 4110, 4194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28496 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2136, 2076, 2069, 1971, 1932, 2059, 2099] payload_rows_per_dst=[4084, 4272, 4152, 4138, 3942, 3864, 4118, 4198] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4272, 4152, 4138, 3942, 3864, 4118, 4198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235487232 dispatch_meta_bytes=114984 combine_payload_bytes=235487232 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235602216 ingress_bytes=2349416448 link_bytes=2349416448 token_bytes=471089448 weight_movement_bytes=2113929216 total_nvlink_bytes=2585018664 weight_to_token_ratio=4.487320 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315575141e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32750 payload_rows_remote=28686 dup_rows=18 dup_rows_remote=6 dedup_ratio=0.000549 router_tokens_per_expert=[2001, 2040, 2034, 2101, 2052, 2068, 2050, 2038] payload_rows_per_dst=[4064, 4068, 4066, 4258, 3982, 4136, 4100, 4076] dup_rows_per_dst=[12, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[4064, 4068, 4066, 4258, 3982, 4136, 4100, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28570 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2036, 2008, 2066, 2112, 2011, 2099, 2066, 1986] payload_rows_per_dst=[4072, 4016, 4132, 4224, 4022, 4198, 4132, 3972] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4072, 4016, 4132, 4224, 4022, 4198, 4132, 3972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233439232 dispatch_meta_bytes=113984 combine_payload_bytes=233439232 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233553216 ingress_bytes=3052011520 link_bytes=3052011520 token_bytes=466992448 weight_movement_bytes=2818572288 total_nvlink_bytes=3285564736 weight_to_token_ratio=6.035584 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708999054e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114736 combine_payload_bytes=234979328 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235094064 ingress_bytes=3758194688 link_bytes=3758194688 token_bytes=470073392 weight_movement_bytes=3523215360 total_nvlink_bytes=3993288752 weight_to_token_ratio=7.495033 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104432151e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234995712 dispatch_meta_bytes=114768 combine_payload_bytes=234995712 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235110480 ingress_bytes=1644281856 link_bytes=1644281856 token_bytes=470106192 weight_movement_bytes=1409286144 total_nvlink_bytes=1879392336 weight_to_token_ratio=2.997804 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207292037e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32718 payload_rows_remote=28492 dup_rows=50 dup_rows_remote=42 dedup_ratio=0.001526 router_tokens_per_expert=[2018, 2081, 2041, 2048, 2076, 2018, 2041, 2061] payload_rows_per_dst=[4036, 4060, 3906, 4174, 4002, 4144, 4170, 4226] dup_rows_per_dst=[0, 0, 0, 6, 2, 22, 12, 8] combine_rows_per_src=[4036, 4060, 3906, 4174, 4002, 4144, 4170, 4226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2027, 2005, 2076, 2076, 2035, 2032, 2065, 2068] payload_rows_per_dst=[4054, 4010, 4152, 4152, 4070, 4064, 4130, 4136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4054, 4010, 4152, 4152, 4070, 4064, 4130, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28664 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2043, 2084, 2066, 2052, 2018, 2059, 2038, 2024] payload_rows_per_dst=[4086, 4168, 4132, 4104, 4036, 4118, 4076, 4048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4086, 4168, 4132, 4104, 4036, 4118, 4076, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28656 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2031, 2064, 2015, 2078, 1998, 2056, 2071] payload_rows_per_dst=[4142, 4062, 4128, 4030, 4156, 3996, 4112, 4142] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4062, 4128, 4030, 4156, 3996, 4112, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234045440 dispatch_meta_bytes=114280 combine_payload_bytes=234045440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234159720 ingress_bytes=1643331584 link_bytes=1643331584 token_bytes=468205160 weight_movement_bytes=1409286144 total_nvlink_bytes=1877491304 weight_to_token_ratio=3.009976 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201970910e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233406464 dispatch_meta_bytes=114136 combine_payload_bytes=233406464 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233520600 ingress_bytes=1642692608 link_bytes=1642692608 token_bytes=466927064 weight_movement_bytes=1409286144 total_nvlink_bytes=1876213208 weight_to_token_ratio=3.018215 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.198392911e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=235208808 ingress_bytes=4462952448 link_bytes=4462952448 token_bytes=470302824 weight_movement_bytes=4227858432 total_nvlink_bytes=4698161256 weight_to_token_ratio=8.989651 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499067078e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234815488 dispatch_meta_bytes=114656 combine_payload_bytes=234815488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234930144 ingress_bytes=1644101632 link_bytes=1644101632 token_bytes=469745632 weight_movement_bytes=1409286144 total_nvlink_bytes=1879031776 weight_to_token_ratio=3.000105 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206282857e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234749952 dispatch_meta_bytes=114624 combine_payload_bytes=234749952 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234864576 ingress_bytes=1644036096 link_bytes=1644036096 token_bytes=469614528 weight_movement_bytes=1409286144 total_nvlink_bytes=1878900672 weight_to_token_ratio=3.000942 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205915883e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28726 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2112, 2021, 2072, 1982, 2049, 2026, 2029, 2093] payload_rows_per_dst=[4224, 4042, 4144, 3964, 4098, 4052, 4058, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4224, 4042, 4144, 3964, 4098, 4052, 4058, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2107, 2044, 2078, 2042, 2068, 2011, 2013] payload_rows_per_dst=[4042, 4214, 4088, 4156, 4084, 4136, 4022, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4214, 4088, 4156, 4084, 4136, 4022, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32624 payload_rows_remote=28450 dup_rows=144 dup_rows_remote=142 dedup_ratio=0.004395 router_tokens_per_expert=[2084, 2084, 2030, 2040, 2046, 1968, 2067, 2065] payload_rows_per_dst=[4174, 4398, 4366, 3928, 4034, 3928, 4008, 3788] dup_rows_per_dst=[2, 76, 44, 22, 0, 0, 0, 0] combine_rows_per_src=[4174, 4398, 4366, 3928, 4034, 3928, 4008, 3788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235323392 dispatch_meta_bytes=114904 combine_payload_bytes=235323392 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235438296 ingress_bytes=3758538752 link_bytes=3758538752 token_bytes=470761688 weight_movement_bytes=3523215360 total_nvlink_bytes=3993977048 weight_to_token_ratio=7.484074 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104624812e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=235061280 ingress_bytes=3758161920 link_bytes=3758161920 token_bytes=470007840 weight_movement_bytes=3523215360 total_nvlink_bytes=3993223200 weight_to_token_ratio=7.496078 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104413802e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2046, 2052, 1987, 2088, 2057, 2072, 2016] payload_rows_per_dst=[4132, 4092, 4104, 3974, 4176, 4114, 4144, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4092, 4104, 3974, 4176, 4114, 4144, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233062400 dispatch_meta_bytes=114368 combine_payload_bytes=233062400 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233176768 ingress_bytes=3051634688 link_bytes=3051634688 token_bytes=466239168 weight_movement_bytes=2818572288 total_nvlink_bytes=3284811456 weight_to_token_ratio=6.045336 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708788044e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32674 payload_rows_remote=28250 dup_rows=94 dup_rows_remote=60 dedup_ratio=0.002869 router_tokens_per_expert=[2060, 2064, 2024, 2050, 2033, 2106, 1961, 2086] payload_rows_per_dst=[4030, 3928, 3758, 4130, 3896, 4382, 4126, 4424] dup_rows_per_dst=[0, 0, 0, 4, 0, 34, 22, 34] combine_rows_per_src=[4030, 3928, 3758, 4130, 3896, 4382, 4126, 4424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2036, 2063, 2030, 2028, 2043, 2040, 2095] payload_rows_per_dst=[4098, 4072, 4126, 4060, 4056, 4086, 4080, 4190] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4072, 4126, 4060, 4056, 4086, 4080, 4190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28718 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2065, 2039, 2030, 2086, 2058, 2025, 2039] payload_rows_per_dst=[4084, 4130, 4078, 4060, 4172, 4116, 4050, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4130, 4078, 4060, 4172, 4116, 4050, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2082, 2031, 2038, 2042, 2066, 2033, 2099, 1993] payload_rows_per_dst=[4164, 4062, 4076, 4084, 4132, 4066, 4198, 3986] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4164, 4062, 4076, 4084, 4132, 4066, 4198, 3986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234848184 ingress_bytes=1644019712 link_bytes=1644019712 token_bytes=469581752 weight_movement_bytes=1409286144 total_nvlink_bytes=1878867896 weight_to_token_ratio=3.001152 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205824140e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=231424000 dispatch_meta_bytes=113240 combine_payload_bytes=231424000 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231537240 ingress_bytes=2345353216 link_bytes=2345353216 token_bytes=462961240 weight_movement_bytes=2113929216 total_nvlink_bytes=2576890456 weight_to_token_ratio=4.566104 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313299901e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235323552 ingress_bytes=2349137920 link_bytes=2349137920 token_bytes=470532256 weight_movement_bytes=2113929216 total_nvlink_bytes=2584461472 weight_to_token_ratio=4.492634 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315419177e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235257856 dispatch_meta_bytes=114872 combine_payload_bytes=235257856 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235372728 ingress_bytes=2349187072 link_bytes=2349187072 token_bytes=470630584 weight_movement_bytes=2113929216 total_nvlink_bytes=2584559800 weight_to_token_ratio=4.491695 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315446700e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2008, 2070, 2068, 2072, 2064, 2035, 2051] payload_rows_per_dst=[4032, 4016, 4140, 4136, 4144, 4128, 4070, 4102] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4016, 4140, 4136, 4144, 4128, 4070, 4102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114736 combine_payload_bytes=234979328 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235094064 ingress_bytes=2348908544 link_bytes=2348908544 token_bytes=470073392 weight_movement_bytes=2113929216 total_nvlink_bytes=2584002608 weight_to_token_ratio=4.497020 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315290736e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2045, 2081, 2053, 2061, 1963, 2077, 2074] payload_rows_per_dst=[4060, 4090, 4162, 4106, 4122, 3926, 4154, 4148] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4090, 4162, 4106, 4122, 3926, 4154, 4148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32656 payload_rows_remote=28206 dup_rows=112 dup_rows_remote=48 dedup_ratio=0.003418 router_tokens_per_expert=[2009, 1997, 2095, 2090, 2011, 2022, 2114, 2046] payload_rows_per_dst=[4450, 4022, 4212, 4178, 4170, 3904, 4074, 3646] dup_rows_per_dst=[64, 12, 16, 0, 18, 2, 0, 0] combine_rows_per_src=[4450, 4022, 4212, 4178, 4170, 3904, 4074, 3646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235651392 ingress_bytes=2349465600 link_bytes=2349465600 token_bytes=471187776 weight_movement_bytes=2113929216 total_nvlink_bytes=2585116992 weight_to_token_ratio=4.486384 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315602664e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2084, 2029, 1996, 2073, 2099, 2062, 2005, 2036] payload_rows_per_dst=[4168, 4058, 3992, 4146, 4198, 4124, 4010, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4168, 4058, 3992, 4146, 4198, 4124, 4010, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=231063552 dispatch_meta_bytes=113016 combine_payload_bytes=231063552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231176568 ingress_bytes=1640349696 link_bytes=1640349696 token_bytes=462240120 weight_movement_bytes=1409286144 total_nvlink_bytes=1871526264 weight_to_token_ratio=3.048818 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.185273582e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32722 payload_rows_remote=28354 dup_rows=46 dup_rows_remote=8 dedup_ratio=0.001404 router_tokens_per_expert=[1973, 2110, 2039, 1987, 2042, 2061, 2084, 2088] payload_rows_per_dst=[3946, 4120, 4078, 4044, 3982, 4122, 4062, 4368] dup_rows_per_dst=[0, 0, 0, 8, 0, 0, 0, 38] combine_rows_per_src=[3946, 4120, 4078, 4044, 3982, 4122, 4062, 4368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114424 combine_payload_bytes=234340352 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234454776 ingress_bytes=2348269568 link_bytes=2348269568 token_bytes=468795128 weight_movement_bytes=2113929216 total_nvlink_bytes=2582724344 weight_to_token_ratio=4.509282 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314932936e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2075, 1981, 2036, 2080, 2102, 2060, 2008] payload_rows_per_dst=[4084, 4150, 3962, 4072, 4160, 4204, 4120, 4016] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4150, 3962, 4072, 4160, 4204, 4120, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 1992, 2066, 2017, 2053, 2049, 2089, 2061] payload_rows_per_dst=[4114, 3984, 4132, 4034, 4106, 4098, 4178, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 3984, 4132, 4034, 4106, 4098, 4178, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2014, 1993, 2043, 2101, 2083, 2070, 2017, 2063] payload_rows_per_dst=[4028, 3986, 4086, 4202, 4166, 4140, 4034, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4028, 3986, 4086, 4202, 4166, 4140, 4034, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=232275968 dispatch_meta_bytes=113448 combine_payload_bytes=232275968 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232389416 ingress_bytes=2346205184 link_bytes=2346205184 token_bytes=464665384 weight_movement_bytes=2113929216 total_nvlink_bytes=2578594600 weight_to_token_ratio=4.549358 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313776968e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234766224 ingress_bytes=2348580864 link_bytes=2348580864 token_bytes=469417872 weight_movement_bytes=2113929216 total_nvlink_bytes=2583347088 weight_to_token_ratio=4.503299 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315107249e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234799008 ingress_bytes=3757899776 link_bytes=3757899776 token_bytes=469483424 weight_movement_bytes=3523215360 total_nvlink_bytes=3992698784 weight_to_token_ratio=7.504451 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104267012e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234913752 ingress_bytes=2348728320 link_bytes=2348728320 token_bytes=469712856 weight_movement_bytes=2113929216 total_nvlink_bytes=2583642072 weight_to_token_ratio=4.500471 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315189818e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28572 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2098, 2042, 2020, 2057, 2059, 2041, 2037] payload_rows_per_dst=[4060, 4196, 4084, 4040, 4114, 4118, 4082, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4196, 4084, 4040, 4114, 4118, 4082, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234126936 ingress_bytes=2347941888 link_bytes=2347941888 token_bytes=468139608 weight_movement_bytes=2113929216 total_nvlink_bytes=2582068824 weight_to_token_ratio=4.515596 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314749449e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32680 payload_rows_remote=28508 dup_rows=88 dup_rows_remote=76 dedup_ratio=0.002686 router_tokens_per_expert=[2020, 1999, 2068, 2090, 2023, 2085, 2012, 2087] payload_rows_per_dst=[4172, 4244, 4308, 3876, 4020, 4058, 3998, 4004] dup_rows_per_dst=[12, 54, 20, 2, 0, 0, 0, 0] combine_rows_per_src=[4172, 4244, 4308, 3876, 4020, 4058, 3998, 4004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2019, 2066, 2057, 2002, 2037, 2048, 2107, 2048] payload_rows_per_dst=[4038, 4132, 4114, 4004, 4074, 4096, 4214, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4038, 4132, 4114, 4004, 4074, 4096, 4214, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234061824 dispatch_meta_bytes=114288 combine_payload_bytes=234061824 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234176112 ingress_bytes=3052634112 link_bytes=3052634112 token_bytes=468237936 weight_movement_bytes=2818572288 total_nvlink_bytes=3286810224 weight_to_token_ratio=6.019530 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709347680e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233537536 dispatch_meta_bytes=114336 combine_payload_bytes=233537536 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233651872 ingress_bytes=2347466752 link_bytes=2347466752 token_bytes=467189408 weight_movement_bytes=2113929216 total_nvlink_bytes=2581118624 weight_to_token_ratio=4.524780 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314483393e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32726 payload_rows_remote=28660 dup_rows=42 dup_rows_remote=40 dedup_ratio=0.001282 router_tokens_per_expert=[2031, 1931, 2079, 2062, 2104, 2072, 2079, 2026] payload_rows_per_dst=[3928, 3862, 3994, 4102, 4350, 4150, 4274, 4066] dup_rows_per_dst=[0, 0, 0, 0, 22, 0, 18, 2] combine_rows_per_src=[3928, 3862, 3994, 4102, 4350, 4150, 4274, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234848184 ingress_bytes=2348662784 link_bytes=2348662784 token_bytes=469581752 weight_movement_bytes=2113929216 total_nvlink_bytes=2583510968 weight_to_token_ratio=4.501728 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315153121e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2097, 2084, 1986, 2063, 2028, 2077, 1982, 2067] payload_rows_per_dst=[4194, 4168, 3972, 4126, 4056, 4154, 3964, 4134] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4168, 3972, 4126, 4056, 4154, 3964, 4134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2000, 2039, 2045, 2080, 2014, 2048, 2051, 2107] payload_rows_per_dst=[4000, 4078, 4090, 4160, 4028, 4096, 4102, 4214] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4000, 4078, 4090, 4160, 4028, 4096, 4102, 4214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2015, 2005, 2021, 2076, 2021, 2096, 2023, 2127] payload_rows_per_dst=[4030, 4010, 4042, 4152, 4042, 4192, 4046, 4254] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4030, 4010, 4042, 4152, 4042, 4192, 4046, 4254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2093, 2043, 2023, 2093, 2029, 2066, 2019, 2018] payload_rows_per_dst=[4186, 4086, 4046, 4186, 4058, 4132, 4038, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4186, 4086, 4046, 4186, 4058, 4132, 4038, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114800 combine_payload_bytes=234782720 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234897520 ingress_bytes=2348711936 link_bytes=2348711936 token_bytes=469680240 weight_movement_bytes=2113929216 total_nvlink_bytes=2583609456 weight_to_token_ratio=4.500784 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315180644e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235323552 ingress_bytes=2349137920 link_bytes=2349137920 token_bytes=470532256 weight_movement_bytes=2113929216 total_nvlink_bytes=2584461472 weight_to_token_ratio=4.492634 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315419177e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234995712 ingress_bytes=3758096384 link_bytes=3758096384 token_bytes=469876736 weight_movement_bytes=3523215360 total_nvlink_bytes=3993092096 weight_to_token_ratio=7.498169 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104377104e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235405512 ingress_bytes=2349219840 link_bytes=2349219840 token_bytes=470696136 weight_movement_bytes=2113929216 total_nvlink_bytes=2584625352 weight_to_token_ratio=4.491070 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315465049e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2009, 2102, 2015, 2084, 1995, 2054, 2059, 2066] payload_rows_per_dst=[4018, 4204, 4030, 4168, 3990, 4108, 4118, 4132] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4018, 4204, 4030, 4168, 3990, 4108, 4118, 4132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32742 payload_rows_remote=28718 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000793 router_tokens_per_expert=[2005, 2050, 2027, 2057, 2103, 2083, 2051, 2008] payload_rows_per_dst=[4024, 4244, 4106, 3878, 4206, 4166, 4102, 4016] dup_rows_per_dst=[0, 12, 14, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 4244, 4106, 3878, 4206, 4166, 4102, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2098, 2084, 2023, 2015, 2029, 2015, 2023, 2097] payload_rows_per_dst=[4196, 4168, 4046, 4030, 4058, 4030, 4046, 4194] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4196, 4168, 4046, 4030, 4058, 4030, 4046, 4194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234143744 dispatch_meta_bytes=114328 combine_payload_bytes=234143744 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234258072 ingress_bytes=3757359104 link_bytes=3757359104 token_bytes=468401816 weight_movement_bytes=3523215360 total_nvlink_bytes=3991617176 weight_to_token_ratio=7.521780 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103964258e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2036, 2001, 2056, 2054, 2101, 2046, 2043, 2047] payload_rows_per_dst=[4072, 4002, 4112, 4108, 4202, 4092, 4086, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4072, 4002, 4112, 4108, 4202, 4092, 4086, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32654 payload_rows_remote=28586 dup_rows=114 dup_rows_remote=100 dedup_ratio=0.003479 router_tokens_per_expert=[2098, 2088, 2039, 2102, 2072, 1986, 2018, 1981] payload_rows_per_dst=[3762, 4064, 4086, 4146, 4140, 4186, 4202, 4068] dup_rows_per_dst=[0, 0, 24, 4, 6, 34, 32, 14] combine_rows_per_src=[3762, 4064, 4086, 4146, 4140, 4186, 4202, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235257856 dispatch_meta_bytes=114976 combine_payload_bytes=235257856 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=235372832 ingress_bytes=5167759360 link_bytes=5167759360 token_bytes=470630688 weight_movement_bytes=4932501504 total_nvlink_bytes=5403132192 weight_to_token_ratio=10.480620 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893729529e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233996288 dispatch_meta_bytes=114256 combine_payload_bytes=233996288 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234110544 ingress_bytes=2347925504 link_bytes=2347925504 token_bytes=468106832 weight_movement_bytes=2113929216 total_nvlink_bytes=2582036048 weight_to_token_ratio=4.515912 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314740275e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235405512 ingress_bytes=2349219840 link_bytes=2349219840 token_bytes=470696136 weight_movement_bytes=2113929216 total_nvlink_bytes=2584625352 weight_to_token_ratio=4.491070 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315465049e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2078, 2075, 2109, 2061, 2000, 2026, 1979] payload_rows_per_dst=[4112, 4156, 4150, 4218, 4122, 4000, 4052, 3958] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4156, 4150, 4218, 4122, 4000, 4052, 3958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28702 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2105, 2016, 2031, 2094, 2031, 2044, 2033, 2030] payload_rows_per_dst=[4210, 4032, 4062, 4188, 4062, 4088, 4066, 4060] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4210, 4032, 4062, 4188, 4062, 4088, 4066, 4060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234126936 ingress_bytes=2347941888 link_bytes=2347941888 token_bytes=468139608 weight_movement_bytes=2113929216 total_nvlink_bytes=2582068824 weight_to_token_ratio=4.515596 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314749449e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234176512 dispatch_meta_bytes=114744 combine_payload_bytes=234176512 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234291256 ingress_bytes=1643462656 link_bytes=1643462656 token_bytes=468467768 weight_movement_bytes=1409286144 total_nvlink_bytes=1877753912 weight_to_token_ratio=3.008288 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202704859e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28524 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2079, 1941, 2073, 2122, 2062, 1999, 2062, 2046] payload_rows_per_dst=[4158, 3882, 4146, 4244, 4124, 3998, 4124, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4158, 3882, 4146, 4244, 4124, 3998, 4124, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235126784 dispatch_meta_bytes=114808 combine_payload_bytes=235126784 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235241592 ingress_bytes=1644412928 link_bytes=1644412928 token_bytes=470368376 weight_movement_bytes=1409286144 total_nvlink_bytes=1879654520 weight_to_token_ratio=2.996133 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208025985e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235667456 dispatch_meta_bytes=115072 combine_payload_bytes=235667456 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235782528 ingress_bytes=3758882816 link_bytes=3758882816 token_bytes=471449984 weight_movement_bytes=3523215360 total_nvlink_bytes=3994665344 weight_to_token_ratio=7.473148 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104817473e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32640 payload_rows_remote=28208 dup_rows=128 dup_rows_remote=88 dedup_ratio=0.003906 router_tokens_per_expert=[2109, 2040, 2027, 2040, 2022, 2031, 2016, 2099] payload_rows_per_dst=[4432, 4084, 4264, 4180, 4044, 4062, 3910, 3664] dup_rows_per_dst=[40, 0, 66, 22, 0, 0, 0, 0] combine_rows_per_src=[4432, 4084, 4264, 4180, 4044, 4062, 3910, 3664] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28682 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2043, 2021, 2039, 2054, 2069, 2060, 2060] payload_rows_per_dst=[4076, 4086, 4042, 4078, 4108, 4138, 4120, 4120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4086, 4042, 4078, 4108, 4138, 4120, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233668608 dispatch_meta_bytes=114096 combine_payload_bytes=233668608 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233782704 ingress_bytes=4461527040 link_bytes=4461527040 token_bytes=467451312 weight_movement_bytes=4227858432 total_nvlink_bytes=4695309744 weight_to_token_ratio=9.044489 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498268909e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2110, 2046, 2104, 2010, 1996, 1976, 2050, 2092] payload_rows_per_dst=[4220, 4092, 4208, 4020, 3992, 3952, 4100, 4184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4220, 4092, 4208, 4020, 3992, 3952, 4100, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1978, 2023, 2088, 2076, 2062, 2110, 2036, 2011] payload_rows_per_dst=[3956, 4046, 4176, 4152, 4124, 4220, 4072, 4022] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3956, 4046, 4176, 4152, 4124, 4220, 4072, 4022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=28506 dup_rows=58 dup_rows_remote=42 dedup_ratio=0.001770 router_tokens_per_expert=[2000, 2029, 2053, 2039, 2109, 2082, 2040, 2032] payload_rows_per_dst=[3920, 3874, 4052, 4146, 4188, 4158, 4168, 4204] dup_rows_per_dst=[0, 0, 0, 10, 12, 0, 20, 16] combine_rows_per_src=[3920, 3874, 4052, 4146, 4188, 4158, 4168, 4204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=231079936 dispatch_meta_bytes=113184 combine_payload_bytes=231079936 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=231193120 ingress_bytes=3049652224 link_bytes=3049652224 token_bytes=462273056 weight_movement_bytes=2818572288 total_nvlink_bytes=3280845344 weight_to_token_ratio=6.097202 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.707677947e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234962944 dispatch_meta_bytes=114728 combine_payload_bytes=234962944 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235077672 ingress_bytes=2348892160 link_bytes=2348892160 token_bytes=470040616 weight_movement_bytes=2113929216 total_nvlink_bytes=2583969832 weight_to_token_ratio=4.497333 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315281562e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2027, 2034, 2061, 2002, 1984, 2091, 2097, 2088] payload_rows_per_dst=[4054, 4068, 4122, 4004, 3968, 4182, 4194, 4176] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4054, 4068, 4122, 4004, 3968, 4182, 4194, 4176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114240 combine_payload_bytes=233963520 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234077760 ingress_bytes=2347892736 link_bytes=2347892736 token_bytes=468041280 weight_movement_bytes=2113929216 total_nvlink_bytes=2581970496 weight_to_token_ratio=4.516544 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314721926e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2062, 2094, 2054, 2042, 2037, 2026, 2029, 2040] payload_rows_per_dst=[4124, 4188, 4108, 4084, 4074, 4052, 4058, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4124, 4188, 4108, 4084, 4074, 4052, 4058, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234766224 ingress_bytes=3053223936 link_bytes=3053223936 token_bytes=469417872 weight_movement_bytes=2818572288 total_nvlink_bytes=3287990160 weight_to_token_ratio=6.004399 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709677956e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1974, 2109, 1964, 2061, 2091, 2062, 2075, 2048] payload_rows_per_dst=[3948, 4218, 3928, 4122, 4182, 4124, 4150, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3948, 4218, 3928, 4122, 4182, 4124, 4150, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=233521152 dispatch_meta_bytes=114192 combine_payload_bytes=233521152 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233635344 ingress_bytes=3052093440 link_bytes=3052093440 token_bytes=467156496 weight_movement_bytes=2818572288 total_nvlink_bytes=3285728784 weight_to_token_ratio=6.033465 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709044926e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234078208 dispatch_meta_bytes=114296 combine_payload_bytes=234078208 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234192504 ingress_bytes=2348007424 link_bytes=2348007424 token_bytes=468270712 weight_movement_bytes=2113929216 total_nvlink_bytes=2582199928 weight_to_token_ratio=4.514331 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314786147e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235356336 ingress_bytes=1644527616 link_bytes=1644527616 token_bytes=470597808 weight_movement_bytes=1409286144 total_nvlink_bytes=1879883952 weight_to_token_ratio=2.994672 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208668190e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234668032 dispatch_meta_bytes=114584 combine_payload_bytes=234668032 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234782616 ingress_bytes=1643954176 link_bytes=1643954176 token_bytes=469450648 weight_movement_bytes=1409286144 total_nvlink_bytes=1878736792 weight_to_token_ratio=3.001990 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205457165e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32726 payload_rows_remote=28542 dup_rows=42 dup_rows_remote=24 dedup_ratio=0.001282 router_tokens_per_expert=[2049, 1981, 2101, 2058, 2019, 2041, 2074, 2061] payload_rows_per_dst=[4184, 4142, 4034, 4074, 3940, 4082, 4148, 4122] dup_rows_per_dst=[18, 24, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4184, 4142, 4034, 4074, 3940, 4082, 4148, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2055, 2090, 2069, 1959, 2051, 2020, 2094] payload_rows_per_dst=[4092, 4110, 4180, 4138, 3918, 4102, 4040, 4188] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4110, 4180, 4138, 3918, 4102, 4040, 4188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28848 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2107, 2119, 1960, 2004, 2058, 2015, 2045, 2076] payload_rows_per_dst=[4214, 4238, 3920, 4008, 4116, 4030, 4090, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4214, 4238, 3920, 4008, 4116, 4030, 4090, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2092, 2109, 1944, 2133, 2082, 2018, 2030, 1976] payload_rows_per_dst=[4184, 4218, 3888, 4266, 4164, 4036, 4060, 3952] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4184, 4218, 3888, 4266, 4164, 4036, 4060, 3952] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=28370 dup_rows=48 dup_rows_remote=18 dedup_ratio=0.001465 router_tokens_per_expert=[2117, 1989, 2027, 2061, 2125, 2010, 2024, 2031] payload_rows_per_dst=[4084, 3978, 3966, 4122, 4076, 3996, 4148, 4350] dup_rows_per_dst=[0, 0, 0, 4, 0, 2, 12, 30] combine_rows_per_src=[4084, 3978, 3966, 4122, 4076, 3996, 4148, 4350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114264 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233930328 ingress_bytes=4461674496 link_bytes=4461674496 token_bytes=467746392 weight_movement_bytes=4227858432 total_nvlink_bytes=4695604824 weight_to_token_ratio=9.038784 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498351478e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234880968 ingress_bytes=3757981696 link_bytes=3757981696 token_bytes=469647304 weight_movement_bytes=3523215360 total_nvlink_bytes=3992862664 weight_to_token_ratio=7.501832 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104312884e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2052, 2048, 2059, 2061, 2072, 2055, 2008, 2029] payload_rows_per_dst=[4104, 4096, 4118, 4122, 4144, 4110, 4016, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4104, 4096, 4118, 4122, 4144, 4110, 4016, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=236322816 dispatch_meta_bytes=115392 combine_payload_bytes=236322816 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=236438208 ingress_bytes=3054895104 link_bytes=3054895104 token_bytes=472761024 weight_movement_bytes=2818572288 total_nvlink_bytes=3291333312 weight_to_token_ratio=5.961939 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710613741e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28472 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2069, 2083, 2148, 2054, 2022, 1985, 1993] payload_rows_per_dst=[4060, 4138, 4166, 4296, 4108, 4044, 3970, 3986] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4138, 4166, 4296, 4108, 4044, 3970, 3986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1998, 2087, 2062, 2015, 2069, 2009, 2055, 2089] payload_rows_per_dst=[3996, 4174, 4124, 4030, 4138, 4018, 4110, 4178] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3996, 4174, 4124, 4030, 4138, 4018, 4110, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28254 dup_rows=62 dup_rows_remote=8 dedup_ratio=0.001892 router_tokens_per_expert=[2014, 2035, 1970, 2054, 2102, 1993, 2098, 2118] payload_rows_per_dst=[4452, 4078, 3940, 3934, 4218, 4024, 3870, 4190] dup_rows_per_dst=[54, 0, 0, 0, 0, 8, 0, 0] combine_rows_per_src=[4452, 4078, 3940, 3934, 4218, 4024, 3870, 4190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234438384 ingress_bytes=1643610112 link_bytes=1643610112 token_bytes=468762352 weight_movement_bytes=1409286144 total_nvlink_bytes=1878048496 weight_to_token_ratio=3.006398 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203530551e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=232407040 dispatch_meta_bytes=113552 combine_payload_bytes=232407040 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232520592 ingress_bytes=1641693184 link_bytes=1641693184 token_bytes=464927632 weight_movement_bytes=1409286144 total_nvlink_bytes=1874213776 weight_to_token_ratio=3.031195 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.192796554e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235651392 ingress_bytes=1644822528 link_bytes=1644822528 token_bytes=471187776 weight_movement_bytes=1409286144 total_nvlink_bytes=1880473920 weight_to_token_ratio=2.990923 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210319574e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233242624 dispatch_meta_bytes=113888 combine_payload_bytes=233242624 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233356512 ingress_bytes=3051814912 link_bytes=3051814912 token_bytes=466599136 weight_movement_bytes=2818572288 total_nvlink_bytes=3285171424 weight_to_token_ratio=6.040672 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708888962e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235635000 ingress_bytes=2349449216 link_bytes=2349449216 token_bytes=471155000 weight_movement_bytes=2113929216 total_nvlink_bytes=2585084216 weight_to_token_ratio=4.486696 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315593490e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=231456768 dispatch_meta_bytes=113048 combine_payload_bytes=231456768 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231569816 ingress_bytes=2345385984 link_bytes=2345385984 token_bytes=463026584 weight_movement_bytes=2113929216 total_nvlink_bytes=2576955800 weight_to_token_ratio=4.565460 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313318250e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2104, 2058, 2008, 1951, 2077, 2035, 2063, 2088] payload_rows_per_dst=[4208, 4116, 4016, 3902, 4154, 4070, 4126, 4176] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4208, 4116, 4016, 3902, 4154, 4070, 4126, 4176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28596 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1983, 2057, 2086, 2029, 2091, 2037, 2088, 2013] payload_rows_per_dst=[3966, 4114, 4172, 4058, 4182, 4074, 4176, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3966, 4114, 4172, 4058, 4182, 4074, 4176, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32690 payload_rows_remote=28550 dup_rows=78 dup_rows_remote=70 dedup_ratio=0.002380 router_tokens_per_expert=[2029, 2106, 2052, 2061, 2030, 2011, 2041, 2054] payload_rows_per_dst=[3710, 3994, 4448, 4152, 4072, 4070, 4104, 4140] dup_rows_per_dst=[0, 0, 56, 4, 2, 6, 2, 8] combine_rows_per_src=[3710, 3994, 4448, 4152, 4072, 4070, 4104, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 1983, 2092, 2062, 2065, 2114, 1984, 2060] payload_rows_per_dst=[4048, 3966, 4184, 4124, 4130, 4228, 3968, 4120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 3966, 4184, 4124, 4130, 4228, 3968, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234831792 ingress_bytes=1644003328 link_bytes=1644003328 token_bytes=469548976 weight_movement_bytes=1409286144 total_nvlink_bytes=1878835120 weight_to_token_ratio=3.001361 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205732396e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234258432 dispatch_meta_bytes=114384 combine_payload_bytes=234258432 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234372816 ingress_bytes=3757473792 link_bytes=3757473792 token_bytes=468631248 weight_movement_bytes=3523215360 total_nvlink_bytes=3991846608 weight_to_token_ratio=7.518097 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104028479e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2025, 2080, 1945, 2049, 2096, 2036, 2065, 2088] payload_rows_per_dst=[4050, 4160, 3890, 4098, 4192, 4072, 4130, 4176] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4050, 4160, 3890, 4098, 4192, 4072, 4130, 4176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 1962, 2105, 2068, 2038, 2044, 2066, 2040] payload_rows_per_dst=[4122, 3924, 4210, 4136, 4076, 4088, 4132, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 3924, 4210, 4136, 4076, 4088, 4132, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28598 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2094, 2057, 2060, 2019, 2020, 2085, 2050, 1999] payload_rows_per_dst=[4188, 4114, 4120, 4038, 4040, 4170, 4100, 3998] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4188, 4114, 4120, 4038, 4040, 4170, 4100, 3998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234717048 ingress_bytes=2348531712 link_bytes=2348531712 token_bytes=469319544 weight_movement_bytes=2113929216 total_nvlink_bytes=2583248760 weight_to_token_ratio=4.504243 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315079726e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=28604 dup_rows=76 dup_rows_remote=76 dedup_ratio=0.002319 router_tokens_per_expert=[2041, 2070, 2043, 2021, 2073, 2017, 2096, 2023] payload_rows_per_dst=[4088, 4522, 4082, 4064, 3826, 4146, 3946, 4018] dup_rows_per_dst=[0, 58, 0, 6, 0, 12, 0, 0] combine_rows_per_src=[4088, 4522, 4082, 4064, 3826, 4146, 3946, 4018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233881600 dispatch_meta_bytes=114480 combine_payload_bytes=233881600 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233996080 ingress_bytes=2347810816 link_bytes=2347810816 token_bytes=467877680 weight_movement_bytes=2113929216 total_nvlink_bytes=2581806896 weight_to_token_ratio=4.518124 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314676055e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234717048 ingress_bytes=3053174784 link_bytes=3053174784 token_bytes=469319544 weight_movement_bytes=2818572288 total_nvlink_bytes=3287891832 weight_to_token_ratio=6.005657 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709650433e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234667872 ingress_bytes=3053125632 link_bytes=3053125632 token_bytes=469221216 weight_movement_bytes=2818572288 total_nvlink_bytes=3287793504 weight_to_token_ratio=6.006916 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709622910e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114392 combine_payload_bytes=234274816 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234389208 ingress_bytes=2348204032 link_bytes=2348204032 token_bytes=468664024 weight_movement_bytes=2113929216 total_nvlink_bytes=2582593240 weight_to_token_ratio=4.510543 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314896239e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114720 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234438688 ingress_bytes=4462182400 link_bytes=4462182400 token_bytes=468762656 weight_movement_bytes=4227858432 total_nvlink_bytes=4696621088 weight_to_token_ratio=9.019188 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498635883e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2023, 2095, 2057, 2043, 2026, 2097, 2046, 1997] payload_rows_per_dst=[4046, 4190, 4114, 4086, 4052, 4194, 4092, 3994] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4046, 4190, 4114, 4086, 4052, 4194, 4092, 3994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2040, 2055, 2028, 1990, 2017, 2112, 2091] payload_rows_per_dst=[4102, 4080, 4110, 4056, 3980, 4034, 4224, 4182] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4080, 4110, 4056, 3980, 4034, 4224, 4182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2080, 2034, 2029, 2073, 2044, 2032, 2018] payload_rows_per_dst=[4148, 4160, 4068, 4058, 4146, 4088, 4064, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4160, 4068, 4058, 4146, 4088, 4064, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32722 payload_rows_remote=28638 dup_rows=46 dup_rows_remote=38 dedup_ratio=0.001404 router_tokens_per_expert=[2116, 2033, 2032, 2063, 2026, 2068, 2035, 2011] payload_rows_per_dst=[4044, 4066, 4042, 3964, 4212, 4222, 4088, 4084] dup_rows_per_dst=[0, 0, 0, 4, 20, 12, 2, 8] combine_rows_per_src=[4044, 4066, 4042, 3964, 4212, 4222, 4088, 4084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114312 combine_payload_bytes=234110976 prefetch_bytes=1409286144 reduce_bytes=4227858432 egress_bytes=234225288 ingress_bytes=5871255552 link_bytes=5871255552 token_bytes=468336264 weight_movement_bytes=5637144576 total_nvlink_bytes=6105480840 weight_to_token_ratio=12.036532 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.287658031e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28640 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2009, 2117, 2045, 2079, 1993, 2064, 2039] payload_rows_per_dst=[4076, 4018, 4234, 4090, 4158, 3986, 4128, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4018, 4234, 4090, 4158, 3986, 4128, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28634 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2017, 2001, 2049, 2067, 2081, 2022, 2104, 2043] payload_rows_per_dst=[4034, 4002, 4098, 4134, 4162, 4044, 4208, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4034, 4002, 4098, 4134, 4162, 4044, 4208, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234880968 ingress_bytes=1644052480 link_bytes=1644052480 token_bytes=469647304 weight_movement_bytes=1409286144 total_nvlink_bytes=1878933448 weight_to_token_ratio=3.000733 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206007627e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114488 combine_payload_bytes=234471424 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234585912 ingress_bytes=4462329856 link_bytes=4462329856 token_bytes=469057336 weight_movement_bytes=4227858432 total_nvlink_bytes=4696915768 weight_to_token_ratio=9.013522 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498718453e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114704 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234717200 ingress_bytes=2348531712 link_bytes=2348531712 token_bytes=469319696 weight_movement_bytes=2113929216 total_nvlink_bytes=2583248912 weight_to_token_ratio=4.504241 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315079726e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28532 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2062, 2056, 2018, 2006, 2118, 2077, 2023] payload_rows_per_dst=[4048, 4124, 4112, 4036, 4012, 4236, 4154, 4046] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4124, 4112, 4036, 4012, 4236, 4154, 4046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=28428 dup_rows=96 dup_rows_remote=92 dedup_ratio=0.002930 router_tokens_per_expert=[2110, 2097, 1991, 2030, 2027, 2069, 2080, 1980] payload_rows_per_dst=[4244, 4278, 3994, 4274, 3916, 3870, 4136, 3960] dup_rows_per_dst=[4, 16, 32, 44, 0, 0, 0, 0] combine_rows_per_src=[4244, 4278, 3994, 4274, 3916, 3870, 4136, 3960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234618880 dispatch_meta_bytes=114560 combine_payload_bytes=234618880 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234733440 ingress_bytes=3053191168 link_bytes=3053191168 token_bytes=469352320 weight_movement_bytes=2818572288 total_nvlink_bytes=3287924608 weight_to_token_ratio=6.005238 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709659608e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234569728 dispatch_meta_bytes=114536 combine_payload_bytes=234569728 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234684264 ingress_bytes=2348498944 link_bytes=2348498944 token_bytes=469253992 weight_movement_bytes=2113929216 total_nvlink_bytes=2583183208 weight_to_token_ratio=4.504872 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315061377e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2097, 2020, 2082, 2042, 2002, 2019, 2059, 2063] payload_rows_per_dst=[4194, 4040, 4164, 4084, 4004, 4038, 4118, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4040, 4164, 4084, 4004, 4038, 4118, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233734144 dispatch_meta_bytes=114128 combine_payload_bytes=233734144 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233848272 ingress_bytes=1643020288 link_bytes=1643020288 token_bytes=467582416 weight_movement_bytes=1409286144 total_nvlink_bytes=1876868560 weight_to_token_ratio=3.013984 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.200227782e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=232882176 dispatch_meta_bytes=114080 combine_payload_bytes=232882176 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=232996256 ingress_bytes=3051454464 link_bytes=3051454464 token_bytes=465878432 weight_movement_bytes=2818572288 total_nvlink_bytes=3284450720 weight_to_token_ratio=6.050017 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708687126e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2086, 2039, 2059, 2078, 2021, 2002, 2080, 2019] payload_rows_per_dst=[4172, 4078, 4118, 4156, 4042, 4004, 4160, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4172, 4078, 4118, 4156, 4042, 4004, 4160, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2066, 2096, 2025, 2020, 2100, 2018, 2030] payload_rows_per_dst=[4058, 4132, 4192, 4050, 4040, 4200, 4036, 4060] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4132, 4192, 4050, 4040, 4200, 4036, 4060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32682 payload_rows_remote=28426 dup_rows=86 dup_rows_remote=50 dedup_ratio=0.002625 router_tokens_per_expert=[2076, 2048, 2038, 2016, 2068, 2064, 2055, 2019] payload_rows_per_dst=[4014, 4160, 4076, 3914, 3874, 4142, 4246, 4256] dup_rows_per_dst=[0, 8, 0, 0, 0, 20, 22, 36] combine_rows_per_src=[4014, 4160, 4076, 3914, 3874, 4142, 4246, 4256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235454688 ingress_bytes=1644625920 link_bytes=1644625920 token_bytes=470794464 weight_movement_bytes=1409286144 total_nvlink_bytes=1880080608 weight_to_token_ratio=2.993421 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209218651e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234700800 dispatch_meta_bytes=114600 combine_payload_bytes=234700800 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234815400 ingress_bytes=3757916160 link_bytes=3757916160 token_bytes=469516200 weight_movement_bytes=3523215360 total_nvlink_bytes=3992731560 weight_to_token_ratio=7.503927 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104276186e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1977, 1984, 2074, 2074, 2102, 2116, 2018, 2039] payload_rows_per_dst=[3954, 3968, 4148, 4148, 4204, 4232, 4036, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3954, 3968, 4148, 4148, 4204, 4232, 4036, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235454688 ingress_bytes=3053912064 link_bytes=3053912064 token_bytes=470794464 weight_movement_bytes=2818572288 total_nvlink_bytes=3289366752 weight_to_token_ratio=5.986842 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710063279e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2038, 2092, 1977, 2118, 1980, 2027, 2115] payload_rows_per_dst=[4074, 4076, 4184, 3954, 4236, 3960, 4054, 4230] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4076, 4184, 3954, 4236, 3960, 4054, 4230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=232865792 dispatch_meta_bytes=113904 combine_payload_bytes=232865792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232979696 ingress_bytes=1642151936 link_bytes=1642151936 token_bytes=465845488 weight_movement_bytes=1409286144 total_nvlink_bytes=1875131632 weight_to_token_ratio=3.025222 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195365374e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2009, 1923, 2071, 2107, 2038, 2075, 2105] payload_rows_per_dst=[4112, 4018, 3846, 4142, 4214, 4076, 4150, 4210] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4018, 3846, 4142, 4214, 4076, 4150, 4210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32664 payload_rows_remote=28422 dup_rows=104 dup_rows_remote=82 dedup_ratio=0.003174 router_tokens_per_expert=[2088, 2065, 1987, 2000, 2136, 2027, 2001, 2080] payload_rows_per_dst=[4242, 4212, 4086, 4042, 4206, 3932, 4028, 3916] dup_rows_per_dst=[22, 24, 18, 24, 0, 14, 2, 0] combine_rows_per_src=[4242, 4212, 4086, 4042, 4206, 3932, 4028, 3916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235487472 ingress_bytes=1644658688 link_bytes=1644658688 token_bytes=470860016 weight_movement_bytes=1409286144 total_nvlink_bytes=1880146160 weight_to_token_ratio=2.993004 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209402138e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=236044288 dispatch_meta_bytes=115256 combine_payload_bytes=236044288 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236159544 ingress_bytes=2349973504 link_bytes=2349973504 token_bytes=472203832 weight_movement_bytes=2113929216 total_nvlink_bytes=2586133048 weight_to_token_ratio=4.476730 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315887070e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2093, 2041, 2048, 2051, 2068, 2026, 2019] payload_rows_per_dst=[4076, 4186, 4082, 4096, 4102, 4136, 4052, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4186, 4082, 4096, 4102, 4136, 4052, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235159632 ingress_bytes=1644331008 link_bytes=1644331008 token_bytes=470204496 weight_movement_bytes=1409286144 total_nvlink_bytes=1879490640 weight_to_token_ratio=2.997177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207567267e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=232833024 dispatch_meta_bytes=114016 combine_payload_bytes=232833024 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=232947040 ingress_bytes=1642119168 link_bytes=1642119168 token_bytes=465780064 weight_movement_bytes=1409286144 total_nvlink_bytes=1875066208 weight_to_token_ratio=3.025647 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195181887e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2019, 2034, 2111, 2059, 2078, 2046, 2016] payload_rows_per_dst=[4042, 4038, 4068, 4222, 4118, 4156, 4092, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4038, 4068, 4222, 4118, 4156, 4092, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2072, 2134, 2089, 2053, 1994, 1967, 2021] payload_rows_per_dst=[4108, 4144, 4268, 4178, 4106, 3988, 3934, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4144, 4268, 4178, 4106, 3988, 3934, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32740 payload_rows_remote=28634 dup_rows=28 dup_rows_remote=26 dedup_ratio=0.000854 router_tokens_per_expert=[2061, 2065, 2032, 2027, 2078, 2041, 2036, 2044] payload_rows_per_dst=[4108, 4100, 4064, 4046, 4062, 4094, 4160, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 20, 2] combine_rows_per_src=[4108, 4100, 4064, 4046, 4062, 4094, 4160, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234143744 dispatch_meta_bytes=114328 combine_payload_bytes=234143744 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234258072 ingress_bytes=1643429888 link_bytes=1643429888 token_bytes=468401816 weight_movement_bytes=1409286144 total_nvlink_bytes=1877687960 weight_to_token_ratio=3.008712 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202521371e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2096, 2009, 2043, 2049, 2037, 2090, 2036, 2024] payload_rows_per_dst=[4192, 4018, 4086, 4098, 4074, 4180, 4072, 4048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4192, 4018, 4086, 4098, 4074, 4180, 4072, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235225200 ingress_bytes=2349039616 link_bytes=2349039616 token_bytes=470335600 weight_movement_bytes=2113929216 total_nvlink_bytes=2584264816 weight_to_token_ratio=4.494512 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315364131e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2082, 2027, 2069, 2066, 2064, 1968, 2060, 2048] payload_rows_per_dst=[4164, 4054, 4138, 4132, 4128, 3936, 4120, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4164, 4054, 4138, 4132, 4128, 3936, 4120, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234913752 ingress_bytes=3053371392 link_bytes=3053371392 token_bytes=469712856 weight_movement_bytes=2818572288 total_nvlink_bytes=3288285144 weight_to_token_ratio=6.000628 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709760526e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234569728 dispatch_meta_bytes=114640 combine_payload_bytes=234569728 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234684368 ingress_bytes=3757785088 link_bytes=3757785088 token_bytes=469254096 weight_movement_bytes=3523215360 total_nvlink_bytes=3992469456 weight_to_token_ratio=7.508119 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104202792e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2106, 1939, 2058, 2029, 2086, 2082, 2033, 2051] payload_rows_per_dst=[4212, 3878, 4116, 4058, 4172, 4164, 4066, 4102] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4212, 3878, 4116, 4058, 4172, 4164, 4066, 4102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32766 payload_rows_remote=28740 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000061 router_tokens_per_expert=[2010, 2092, 2034, 2105, 2080, 2004, 2024, 2035] payload_rows_per_dst=[4026, 4192, 4076, 4194, 4152, 4008, 4048, 4070] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4026, 4192, 4076, 4194, 4152, 4008, 4048, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234979320 ingress_bytes=3053436928 link_bytes=3053436928 token_bytes=469843960 weight_movement_bytes=2818572288 total_nvlink_bytes=3288416248 weight_to_token_ratio=5.998954 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709797223e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234799008 ingress_bytes=1643970560 link_bytes=1643970560 token_bytes=469483424 weight_movement_bytes=1409286144 total_nvlink_bytes=1878769568 weight_to_token_ratio=3.001780 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205548909e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7512423080 prefetch_bytes=21139292160 reduce_bytes=21843935232 weight_movement_bytes=42983227392 egress_bytes=3757128360 ingress_bytes=46738522112 link_bytes=46738522112 total_nvlink_bytes=50495650472 weight_to_token_ratio=5.721620 prefetch_share_of_ingress=0.452288 dedup_ratio=0.000000 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=31 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.635726339e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28736 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1975, 2016, 2052, 2056, 2073, 2114, 2034, 2064] payload_rows_per_dst=[3950, 4032, 4104, 4112, 4146, 4228, 4068, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3950, 4032, 4104, 4112, 4146, 4228, 4068, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234438384 ingress_bytes=3052896256 link_bytes=3052896256 token_bytes=468762352 weight_movement_bytes=2818572288 total_nvlink_bytes=3287334640 weight_to_token_ratio=6.012796 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709494469e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114968 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235553048 ingress_bytes=3054010368 link_bytes=3054010368 token_bytes=470991128 weight_movement_bytes=2818572288 total_nvlink_bytes=3289563416 weight_to_token_ratio=5.984343 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710118325e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28794 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2003, 2026, 1987, 2060, 2053, 2071, 2059, 2125] payload_rows_per_dst=[4006, 4052, 3974, 4120, 4106, 4142, 4118, 4250] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4006, 4052, 3974, 4120, 4106, 4142, 4118, 4250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32658 payload_rows_remote=28448 dup_rows=110 dup_rows_remote=78 dedup_ratio=0.003357 router_tokens_per_expert=[2046, 2047, 2041, 2123, 2013, 2058, 2037, 2019] payload_rows_per_dst=[3970, 3836, 4004, 3970, 4040, 4374, 4254, 4210] dup_rows_per_dst=[0, 0, 0, 0, 4, 36, 38, 32] combine_rows_per_src=[3970, 3836, 4004, 3970, 4040, 4374, 4254, 4210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235405312 dispatch_meta_bytes=114944 combine_payload_bytes=235405312 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235520256 ingress_bytes=2349334528 link_bytes=2349334528 token_bytes=470925568 weight_movement_bytes=2113929216 total_nvlink_bytes=2584854784 weight_to_token_ratio=4.488882 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315529270e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7470899400 prefetch_bytes=17616076800 reduce_bytes=27481079808 weight_movement_bytes=45097156608 egress_bytes=3736363208 ingress_bytes=48831692800 link_bytes=48831692800 total_nvlink_bytes=52568056008 weight_to_token_ratio=6.036376 prefetch_share_of_ingress=0.360751 dedup_ratio=0.002171 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=39 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.708981852e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2008, 2051, 2092, 2006, 2027, 2082, 2044, 2074] payload_rows_per_dst=[4016, 4102, 4184, 4012, 4054, 4164, 4088, 4148] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4016, 4102, 4184, 4012, 4054, 4164, 4088, 4148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235880448 dispatch_meta_bytes=115176 combine_payload_bytes=235880448 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235995624 ingress_bytes=1645166592 link_bytes=1645166592 token_bytes=471876072 weight_movement_bytes=1409286144 total_nvlink_bytes=1881162216 weight_to_token_ratio=2.986560 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212246189e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2095, 2109, 1976, 2100, 1986, 2047, 2065, 2006] payload_rows_per_dst=[4190, 4218, 3952, 4200, 3972, 4094, 4130, 4012] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4190, 4218, 3952, 4200, 3972, 4094, 4130, 4012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233046016 dispatch_meta_bytes=114104 combine_payload_bytes=233046016 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233160120 ingress_bytes=1642332160 link_bytes=1642332160 token_bytes=466206136 weight_movement_bytes=1409286144 total_nvlink_bytes=1875492280 weight_to_token_ratio=3.022882 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196374553e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2013, 2072, 2053, 2072, 2003, 2018, 2072, 2081] payload_rows_per_dst=[4026, 4144, 4106, 4144, 4006, 4036, 4144, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4026, 4144, 4106, 4144, 4006, 4036, 4144, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=235684176 ingress_bytes=4463427584 link_bytes=4463427584 token_bytes=471253328 weight_movement_bytes=4227858432 total_nvlink_bytes=4699111760 weight_to_token_ratio=8.971520 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499333134e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234717048 ingress_bytes=1643888640 link_bytes=1643888640 token_bytes=469319544 weight_movement_bytes=1409286144 total_nvlink_bytes=1878605688 weight_to_token_ratio=3.002829 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205090191e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7469785024 prefetch_bytes=22548578304 reduce_bytes=12683575296 weight_movement_bytes=35232153600 egress_bytes=3735805888 ingress_bytes=38966132736 link_bytes=38966132736 total_nvlink_bytes=42701938624 weight_to_token_ratio=4.716622 prefetch_share_of_ingress=0.578671 dedup_ratio=0.002277 prefetch_slots=64 unused_prefetch_slots=0 slot_utilization=1.000000 reduce_slots=18 remote_demand=32 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.015152 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.363712988e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2070, 2077, 2064, 1963, 2038, 2078, 2067, 2027] payload_rows_per_dst=[4140, 4154, 4128, 3926, 4076, 4156, 4134, 4054] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4154, 4128, 3926, 4076, 4156, 4134, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7512488632 prefetch_bytes=18320719872 reduce_bytes=23957864448 weight_movement_bytes=42278584320 egress_bytes=3757161144 ingress_bytes=46033911808 link_bytes=46033911808 total_nvlink_bytes=49791072952 weight_to_token_ratio=5.627773 prefetch_share_of_ingress=0.397983 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=34 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.611066816e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235487472 ingress_bytes=3053944832 link_bytes=3053944832 token_bytes=470860016 weight_movement_bytes=2818572288 total_nvlink_bytes=3289432304 weight_to_token_ratio=5.986009 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710081628e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2113, 2011, 2044, 2078, 2090, 1980, 2031, 2037] payload_rows_per_dst=[4226, 4022, 4088, 4156, 4180, 3960, 4062, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4226, 4022, 4088, 4156, 4180, 3960, 4062, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7520879288 prefetch_bytes=19730006016 reduce_bytes=14797504512 weight_movement_bytes=34527510528 egress_bytes=3761357496 ingress_bytes=38287032320 link_bytes=38287032320 total_nvlink_bytes=42048389816 weight_to_token_ratio=4.590887 prefetch_share_of_ingress=0.515318 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=21 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.339946245e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234520344 ingress_bytes=3052978176 link_bytes=3052978176 token_bytes=468926232 weight_movement_bytes=2818572288 total_nvlink_bytes=3287498520 weight_to_token_ratio=6.010694 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709540341e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235061280 ingress_bytes=1644232704 link_bytes=1644232704 token_bytes=470007840 weight_movement_bytes=1409286144 total_nvlink_bytes=1879293984 weight_to_token_ratio=2.998431 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207016806e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7511603680 prefetch_bytes=20434649088 reduce_bytes=19730006016 weight_movement_bytes=40164655104 egress_bytes=3756718560 ingress_bytes=43919540224 link_bytes=43919540224 total_nvlink_bytes=47676258784 weight_to_token_ratio=5.347015 prefetch_share_of_ingress=0.465275 dedup_ratio=0.000000 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=28 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.537069327e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28780 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2049, 2034, 2066, 2092, 1994, 2040, 2063] payload_rows_per_dst=[4092, 4098, 4068, 4132, 4184, 3988, 4080, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4098, 4068, 4132, 4184, 3988, 4080, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7520158216 prefetch_bytes=16206790656 reduce_bytes=22548578304 weight_movement_bytes=38755368960 egress_bytes=3760996872 ingress_bytes=42514530304 link_bytes=42514530304 total_nvlink_bytes=46275527176 weight_to_token_ratio=5.153531 prefetch_share_of_ingress=0.381206 dedup_ratio=0.000000 prefetch_slots=46 unused_prefetch_slots=18 slot_utilization=0.718750 reduce_slots=32 remote_demand=23 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.487897645e-04 prefetch_bytes_per_gemm_flop=5.671953914e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235765760 dispatch_meta_bytes=115120 combine_payload_bytes=235765760 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235880880 ingress_bytes=1645051904 link_bytes=1645051904 token_bytes=471646640 weight_movement_bytes=1409286144 total_nvlink_bytes=1880932784 weight_to_token_ratio=2.988013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211603984e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7514750176 prefetch_bytes=19730006016 reduce_bytes=12683575296 weight_movement_bytes=32413581312 egress_bytes=3758292192 ingress_bytes=36170039296 link_bytes=36170039296 total_nvlink_bytes=39928331488 weight_to_token_ratio=4.313328 prefetch_share_of_ingress=0.545479 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=18 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.265857012e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none - [2026-09-16 21:35:52] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 580.7 | throughput per GPU (TFLOP/s/GPU): 574.1 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.065318E+01 | loss scale: 1.0 | grad norm: 43.754 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32696 payload_rows_remote=28518 dup_rows=72 dup_rows_remote=68 dedup_ratio=0.002197 router_tokens_per_expert=[2078, 2006, 2028, 2017, 2098, 2088, 2085, 1984] payload_rows_per_dst=[4178, 4266, 4146, 4206, 3964, 4176, 4022, 3738] dup_rows_per_dst=[4, 22, 10, 36, 0, 0, 0, 0] combine_rows_per_src=[4178, 4266, 4146, 4206, 3964, 4176, 4022, 3738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233619456 dispatch_meta_bytes=114344 combine_payload_bytes=233619456 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233733800 ingress_bytes=3052191744 link_bytes=3052191744 token_bytes=467353256 weight_movement_bytes=2818572288 total_nvlink_bytes=3285925544 weight_to_token_ratio=6.030925 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709099972e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28668 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2026, 2018, 2083, 2078, 2050, 2037, 2041, 2051] payload_rows_per_dst=[4052, 4036, 4166, 4156, 4100, 4074, 4082, 4102] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4052, 4036, 4166, 4156, 4100, 4074, 4082, 4102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=28358 dup_rows=66 dup_rows_remote=20 dedup_ratio=0.002014 router_tokens_per_expert=[2065, 2011, 2058, 2006, 2029, 2042, 2124, 2049] payload_rows_per_dst=[4086, 4022, 4116, 4012, 4040, 3768, 4314, 4344] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 20, 46] combine_rows_per_src=[4086, 4022, 4116, 4012, 4040, 3768, 4314, 4344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2057, 2103, 2026, 1982, 2084, 2040, 2047] payload_rows_per_dst=[4090, 4114, 4206, 4052, 3964, 4168, 4080, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4114, 4206, 4052, 3964, 4168, 4080, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 1972, 2064, 2042, 2049, 2136, 2050, 1991] payload_rows_per_dst=[4160, 3944, 4128, 4084, 4098, 4272, 4100, 3982] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 3944, 4128, 4084, 4098, 4272, 4100, 3982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28788 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2012, 2048, 1990, 2027, 2092, 2048, 2045, 2122] payload_rows_per_dst=[4024, 4096, 3980, 4054, 4184, 4096, 4090, 4244] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 4096, 3980, 4054, 4184, 4096, 4090, 4244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2066, 1959, 2099, 2103, 2055, 1998, 2075] payload_rows_per_dst=[4058, 4132, 3918, 4198, 4206, 4110, 3996, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4132, 3918, 4198, 4206, 4110, 3996, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 2, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2068, 2049, 1985, 2106, 2052, 2032, 2034] payload_rows_per_dst=[4116, 4136, 4098, 3970, 4212, 4104, 4064, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4136, 4098, 3970, 4212, 4104, 4064, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32644 payload_rows_remote=28618 dup_rows=124 dup_rows_remote=124 dedup_ratio=0.003784 router_tokens_per_expert=[2010, 2043, 2038, 2012, 2007, 2093, 2061, 2120] payload_rows_per_dst=[4026, 4266, 4198, 4372, 4014, 4090, 3960, 3718] dup_rows_per_dst=[0, 42, 20, 62, 0, 0, 0, 0] combine_rows_per_src=[4026, 4266, 4198, 4372, 4014, 4090, 3960, 3718] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234848256 dispatch_meta_bytes=114672 combine_payload_bytes=234848256 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234962928 ingress_bytes=4462706688 link_bytes=4462706688 token_bytes=469811184 weight_movement_bytes=4227858432 total_nvlink_bytes=4697669616 weight_to_token_ratio=8.999059 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498929463e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=232308736 dispatch_meta_bytes=113512 combine_payload_bytes=232308736 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232422248 ingress_bytes=3050881024 link_bytes=3050881024 token_bytes=464730984 weight_movement_bytes=2818572288 total_nvlink_bytes=3283303272 weight_to_token_ratio=6.064955 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708366023e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234848184 ingress_bytes=1644019712 link_bytes=1644019712 token_bytes=469581752 weight_movement_bytes=1409286144 total_nvlink_bytes=1878867896 weight_to_token_ratio=3.001152 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205824140e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114736 combine_payload_bytes=234979328 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235094064 ingress_bytes=1644265472 link_bytes=1644265472 token_bytes=470073392 weight_movement_bytes=1409286144 total_nvlink_bytes=1879359536 weight_to_token_ratio=2.998013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207200293e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235831296 dispatch_meta_bytes=115152 combine_payload_bytes=235831296 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235946448 ingress_bytes=1645117440 link_bytes=1645117440 token_bytes=471777744 weight_movement_bytes=1409286144 total_nvlink_bytes=1881063888 weight_to_token_ratio=2.987182 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211970958e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234880968 ingress_bytes=3053338624 link_bytes=3053338624 token_bytes=469647304 weight_movement_bytes=2818572288 total_nvlink_bytes=3288219592 weight_to_token_ratio=6.001466 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709742177e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234438656 dispatch_meta_bytes=114968 combine_payload_bytes=234438656 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234553624 ingress_bytes=2348367872 link_bytes=2348367872 token_bytes=468992280 weight_movement_bytes=2113929216 total_nvlink_bytes=2582921496 weight_to_token_ratio=4.507386 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314987983e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235257984 ingress_bytes=3053715456 link_bytes=3053715456 token_bytes=470401152 weight_movement_bytes=2818572288 total_nvlink_bytes=3288973440 weight_to_token_ratio=5.991848 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709953187e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28804 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2110, 2027, 2103, 2062, 1982, 2046, 2048, 2006] payload_rows_per_dst=[4220, 4054, 4206, 4124, 3964, 4092, 4096, 4012] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4220, 4054, 4206, 4124, 3964, 4092, 4096, 4012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28390 dup_rows=62 dup_rows_remote=32 dedup_ratio=0.001892 router_tokens_per_expert=[2047, 2040, 2068, 1969, 2101, 2062, 2059, 2038] payload_rows_per_dst=[4316, 4106, 4114, 4166, 4208, 4070, 3664, 4062] dup_rows_per_dst=[30, 2, 0, 28, 2, 0, 0, 0] combine_rows_per_src=[4316, 4106, 4114, 4166, 4208, 4070, 3664, 4062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=28456 dup_rows=60 dup_rows_remote=46 dedup_ratio=0.001831 router_tokens_per_expert=[2142, 2038, 1983, 2032, 2023, 2021, 2088, 2057] payload_rows_per_dst=[4272, 4076, 3924, 3844, 4264, 4020, 4056, 4252] dup_rows_per_dst=[0, 0, 0, 0, 44, 0, 2, 14] combine_rows_per_src=[4272, 4076, 3924, 3844, 4264, 4020, 4056, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2022, 2066, 2002, 2040, 2054, 2031, 2150, 2019] payload_rows_per_dst=[4044, 4132, 4004, 4080, 4108, 4062, 4300, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4044, 4132, 4004, 4080, 4108, 4062, 4300, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28710 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2043, 2029, 2051, 2074, 2047, 2042, 2018, 2080] payload_rows_per_dst=[4086, 4058, 4102, 4148, 4094, 4084, 4036, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4086, 4058, 4102, 4148, 4094, 4084, 4036, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28548 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1988, 1990, 2089, 1969, 2100, 2110, 2045, 2093] payload_rows_per_dst=[3976, 3980, 4178, 3938, 4200, 4220, 4090, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3976, 3980, 4178, 3938, 4200, 4220, 4090, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28626 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2048, 2007, 2071, 2042, 2055, 2004, 2052, 2105] payload_rows_per_dst=[4096, 4014, 4142, 4084, 4110, 4008, 4104, 4210] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 4014, 4142, 4084, 4110, 4008, 4104, 4210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235962368 dispatch_meta_bytes=115216 combine_payload_bytes=235962368 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236077584 ingress_bytes=1645248512 link_bytes=1645248512 token_bytes=472039952 weight_movement_bytes=1409286144 total_nvlink_bytes=1881326096 weight_to_token_ratio=2.985523 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212704907e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=232570880 dispatch_meta_bytes=113688 combine_payload_bytes=232570880 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232684568 ingress_bytes=1641857024 link_bytes=1641857024 token_bytes=465255448 weight_movement_bytes=1409286144 total_nvlink_bytes=1874541592 weight_to_token_ratio=3.029059 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.193713990e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2073, 1997, 2058, 1985, 2096, 2062, 2089] payload_rows_per_dst=[4048, 4146, 3994, 4116, 3970, 4192, 4124, 4178] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4146, 3994, 4116, 3970, 4192, 4124, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233111552 dispatch_meta_bytes=114008 combine_payload_bytes=233111552 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233225560 ingress_bytes=3051683840 link_bytes=3051683840 token_bytes=466337112 weight_movement_bytes=2818572288 total_nvlink_bytes=3284909400 weight_to_token_ratio=6.044066 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708815567e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235126848 ingress_bytes=2348941312 link_bytes=2348941312 token_bytes=470138944 weight_movement_bytes=2113929216 total_nvlink_bytes=2584068160 weight_to_token_ratio=4.496392 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315309085e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235192320 dispatch_meta_bytes=114840 combine_payload_bytes=235192320 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235307160 ingress_bytes=1644478464 link_bytes=1644478464 token_bytes=470499480 weight_movement_bytes=1409286144 total_nvlink_bytes=1879785624 weight_to_token_ratio=2.995298 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208392959e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233865216 dispatch_meta_bytes=114192 combine_payload_bytes=233865216 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=233979408 ingress_bytes=5166366720 link_bytes=5166366720 token_bytes=467844624 weight_movement_bytes=4932501504 total_nvlink_bytes=5400346128 weight_to_token_ratio=10.543033 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.892949708e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234504192 dispatch_meta_bytes=114504 combine_payload_bytes=234504192 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234618696 ingress_bytes=2348433408 link_bytes=2348433408 token_bytes=469122888 weight_movement_bytes=2113929216 total_nvlink_bytes=2583052104 weight_to_token_ratio=4.506131 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315024680e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234766224 ingress_bytes=3053223936 link_bytes=3053223936 token_bytes=469417872 weight_movement_bytes=2818572288 total_nvlink_bytes=3287990160 weight_to_token_ratio=6.004399 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709677956e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28664 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2012, 1965, 2070, 2016, 2052, 2087, 2148, 2034] payload_rows_per_dst=[4024, 3930, 4140, 4032, 4104, 4174, 4296, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 3930, 4140, 4032, 4104, 4174, 4296, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=28540 dup_rows=64 dup_rows_remote=58 dedup_ratio=0.001953 router_tokens_per_expert=[2064, 2043, 2043, 2025, 1997, 2047, 2072, 2093] payload_rows_per_dst=[4164, 4060, 4288, 4134, 3902, 3826, 4144, 4186] dup_rows_per_dst=[6, 0, 16, 38, 4, 0, 0, 0] combine_rows_per_src=[4164, 4060, 4288, 4134, 3902, 3826, 4144, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=28516 dup_rows=34 dup_rows_remote=16 dedup_ratio=0.001038 router_tokens_per_expert=[2069, 2044, 2027, 2068, 1981, 2027, 2101, 2067] payload_rows_per_dst=[4138, 4064, 3950, 4136, 3878, 3952, 4398, 4218] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 16, 18] combine_rows_per_src=[4138, 4064, 3950, 4136, 3878, 3952, 4398, 4218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28592 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2007, 2065, 2058, 2088, 1980, 2057, 2094, 2035] payload_rows_per_dst=[4014, 4130, 4116, 4176, 3960, 4114, 4188, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4014, 4130, 4116, 4176, 3960, 4114, 4188, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 2026, 2072, 1996, 2084, 2064, 2047, 2064] payload_rows_per_dst=[4062, 4052, 4144, 3992, 4168, 4128, 4094, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 4052, 4144, 3992, 4168, 4128, 4094, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1996, 2088, 2101, 2021, 2027, 2097, 1990, 2064] payload_rows_per_dst=[3992, 4176, 4202, 4042, 4054, 4194, 3980, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3992, 4176, 4202, 4042, 4054, 4194, 3980, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233799680 dispatch_meta_bytes=114392 combine_payload_bytes=233799680 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233914072 ingress_bytes=3052371968 link_bytes=3052371968 token_bytes=467713752 weight_movement_bytes=2818572288 total_nvlink_bytes=3286286040 weight_to_token_ratio=6.026276 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709200890e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234815488 dispatch_meta_bytes=114656 combine_payload_bytes=234815488 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234930144 ingress_bytes=2348744704 link_bytes=2348744704 token_bytes=469745632 weight_movement_bytes=2113929216 total_nvlink_bytes=2583674848 weight_to_token_ratio=4.500157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315198993e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1995, 2101, 2008, 2042, 2063, 2082, 2073, 2020] payload_rows_per_dst=[3990, 4202, 4016, 4084, 4126, 4164, 4146, 4040] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3990, 4202, 4016, 4084, 4126, 4164, 4146, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28782 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2101, 2053, 2043, 2056, 2134, 1986, 1993, 2018] payload_rows_per_dst=[4202, 4106, 4086, 4112, 4268, 3972, 3986, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4202, 4106, 4086, 4112, 4268, 3972, 3986, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233603072 dispatch_meta_bytes=114128 combine_payload_bytes=233603072 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233717200 ingress_bytes=2347532288 link_bytes=2347532288 token_bytes=467320272 weight_movement_bytes=2113929216 total_nvlink_bytes=2581249488 weight_to_token_ratio=4.523513 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314520090e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234225664 dispatch_meta_bytes=114368 combine_payload_bytes=234225664 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234340032 ingress_bytes=1643511808 link_bytes=1643511808 token_bytes=468565696 weight_movement_bytes=1409286144 total_nvlink_bytes=1877851840 weight_to_token_ratio=3.007660 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202980089e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235356336 ingress_bytes=2349170688 link_bytes=2349170688 token_bytes=470597808 weight_movement_bytes=2113929216 total_nvlink_bytes=2584527024 weight_to_token_ratio=4.492008 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315437526e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234078208 dispatch_meta_bytes=114296 combine_payload_bytes=234078208 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234192504 ingress_bytes=4461936640 link_bytes=4461936640 token_bytes=468270712 weight_movement_bytes=4227858432 total_nvlink_bytes=4696129144 weight_to_token_ratio=9.028663 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498498268e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235651392 ingress_bytes=3758751744 link_bytes=3758751744 token_bytes=471187776 weight_movement_bytes=3523215360 total_nvlink_bytes=3994403136 weight_to_token_ratio=7.477306 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104744079e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235782144 dispatch_meta_bytes=115128 combine_payload_bytes=235782144 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235897272 ingress_bytes=3054354432 link_bytes=3054354432 token_bytes=471679416 weight_movement_bytes=2818572288 total_nvlink_bytes=3290251704 weight_to_token_ratio=5.975610 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710310987e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32662 payload_rows_remote=28238 dup_rows=106 dup_rows_remote=62 dedup_ratio=0.003235 router_tokens_per_expert=[2092, 2078, 2013, 2022, 2020, 2049, 2023, 2087] payload_rows_per_dst=[4424, 3976, 4304, 4088, 4048, 4100, 3580, 4142] dup_rows_per_dst=[44, 0, 50, 2, 10, 0, 0, 0] combine_rows_per_src=[4424, 3976, 4304, 4088, 4048, 4100, 3580, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2190, 2030, 2024, 2054, 2067, 1957, 2001] payload_rows_per_dst=[4122, 4380, 4060, 4048, 4108, 4134, 3914, 4002] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4380, 4060, 4048, 4108, 4134, 3914, 4002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32668 payload_rows_remote=28262 dup_rows=100 dup_rows_remote=26 dedup_ratio=0.003052 router_tokens_per_expert=[2010, 1998, 2055, 2045, 2091, 2060, 2132, 1993] payload_rows_per_dst=[3800, 3956, 3914, 4200, 4128, 4122, 4142, 4406] dup_rows_per_dst=[0, 0, 0, 16, 0, 0, 10, 74] combine_rows_per_src=[3800, 3956, 3914, 4200, 4128, 4122, 4142, 4406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28840 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 2046, 2022, 1964, 2119, 2143, 2036, 2019] payload_rows_per_dst=[4070, 4092, 4044, 3928, 4238, 4286, 4072, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 4092, 4044, 3928, 4238, 4286, 4072, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=231325696 dispatch_meta_bytes=113200 combine_payload_bytes=231325696 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231438896 ingress_bytes=2345254912 link_bytes=2345254912 token_bytes=462764592 weight_movement_bytes=2113929216 total_nvlink_bytes=2576693808 weight_to_token_ratio=4.568044 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313244855e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28820 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2094, 1974, 2103, 2095, 2008, 2022, 2067, 2021] payload_rows_per_dst=[4188, 3948, 4206, 4190, 4016, 4044, 4134, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4188, 3948, 4206, 4190, 4016, 4044, 4134, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2084, 2022, 2035, 1987, 2116, 2067, 2073, 2000] payload_rows_per_dst=[4168, 4044, 4070, 3974, 4232, 4134, 4146, 4000] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4168, 4044, 4070, 3974, 4232, 4134, 4146, 4000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2101, 2018, 2030, 2022, 2034, 2097, 2018] payload_rows_per_dst=[4128, 4202, 4036, 4060, 4044, 4068, 4194, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4202, 4036, 4060, 4044, 4068, 4194, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114640 combine_payload_bytes=234782720 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234897360 ingress_bytes=3053355008 link_bytes=3053355008 token_bytes=469680080 weight_movement_bytes=2818572288 total_nvlink_bytes=3288252368 weight_to_token_ratio=6.001047 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709751351e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2043, 2067, 2090, 2048, 2077, 2006, 1992] payload_rows_per_dst=[4122, 4086, 4134, 4180, 4096, 4154, 4012, 3984] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4086, 4134, 4180, 4096, 4154, 4012, 3984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=231522304 dispatch_meta_bytes=113152 combine_payload_bytes=231522304 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231635456 ingress_bytes=1640808448 link_bytes=1640808448 token_bytes=463157760 weight_movement_bytes=1409286144 total_nvlink_bytes=1872443904 weight_to_token_ratio=3.042778 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.187842402e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=236257280 dispatch_meta_bytes=115360 combine_payload_bytes=236257280 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236372640 ingress_bytes=1645543424 link_bytes=1645543424 token_bytes=472629920 weight_movement_bytes=1409286144 total_nvlink_bytes=1881916064 weight_to_token_ratio=2.981796 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.214356291e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=236093440 dispatch_meta_bytes=115280 combine_payload_bytes=236093440 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=236208720 ingress_bytes=3054665728 link_bytes=3054665728 token_bytes=472302160 weight_movement_bytes=2818572288 total_nvlink_bytes=3290874448 weight_to_token_ratio=5.967731 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710485300e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235208808 ingress_bytes=2349023232 link_bytes=2349023232 token_bytes=470302824 weight_movement_bytes=2113929216 total_nvlink_bytes=2584232040 weight_to_token_ratio=4.494826 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315354957e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=235225200 ingress_bytes=4462968832 link_bytes=4462968832 token_bytes=470335600 weight_movement_bytes=4227858432 total_nvlink_bytes=4698194032 weight_to_token_ratio=8.989025 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499076252e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235684176 ingress_bytes=2349498368 link_bytes=2349498368 token_bytes=471253328 weight_movement_bytes=2113929216 total_nvlink_bytes=2585182544 weight_to_token_ratio=4.485760 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315621013e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=28536 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.002930 router_tokens_per_expert=[2064, 2031, 2082, 1975, 2108, 2002, 2099, 2023] payload_rows_per_dst=[4136, 4328, 4264, 3950, 4354, 3664, 4030, 3946] dup_rows_per_dst=[0, 52, 18, 0, 26, 0, 0, 0] combine_rows_per_src=[4136, 4328, 4264, 3950, 4354, 3664, 4030, 3946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28754 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2001, 2105, 2035, 2007, 2007, 2061, 2166, 2002] payload_rows_per_dst=[4002, 4210, 4070, 4014, 4014, 4122, 4332, 4004] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4002, 4210, 4070, 4014, 4014, 4122, 4332, 4004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233766912 dispatch_meta_bytes=114528 combine_payload_bytes=233766912 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233881440 ingress_bytes=2347696128 link_bytes=2347696128 token_bytes=467648352 weight_movement_bytes=2113929216 total_nvlink_bytes=2581577568 weight_to_token_ratio=4.520339 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314611834e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 1977, 2022, 2068, 1987, 2025, 2088, 2126] payload_rows_per_dst=[4182, 3954, 4044, 4136, 3974, 4050, 4176, 4252] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 3954, 4044, 4136, 3974, 4050, 4176, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32714 payload_rows_remote=28658 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.001648 router_tokens_per_expert=[2021, 2061, 2048, 2004, 2133, 1981, 2108, 2028] payload_rows_per_dst=[3850, 4278, 3998, 4008, 4266, 4042, 4216, 4056] dup_rows_per_dst=[0, 36, 0, 0, 0, 18, 0, 0] combine_rows_per_src=[3850, 4278, 3998, 4008, 4266, 4042, 4216, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28802 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 1983, 2039, 2078, 2047, 2067, 2086, 2038] payload_rows_per_dst=[4092, 3966, 4078, 4156, 4094, 4134, 4172, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 3966, 4078, 4156, 4094, 4134, 4172, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2095, 2023, 2023, 2084, 2063, 2031, 2020] payload_rows_per_dst=[4090, 4190, 4046, 4046, 4168, 4126, 4062, 4040] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4190, 4046, 4046, 4168, 4126, 4062, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235552768 dispatch_meta_bytes=115016 combine_payload_bytes=235552768 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235667784 ingress_bytes=2349481984 link_bytes=2349481984 token_bytes=471220552 weight_movement_bytes=2113929216 total_nvlink_bytes=2585149768 weight_to_token_ratio=4.486072 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315611839e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28618 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1966, 2034, 2045, 2085, 2027, 2075, 2060, 2092] payload_rows_per_dst=[3932, 4068, 4090, 4170, 4054, 4150, 4120, 4184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3932, 4068, 4090, 4170, 4054, 4150, 4120, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2076, 2045, 2044, 2074, 2070, 2013, 2055, 2007] payload_rows_per_dst=[4152, 4090, 4088, 4148, 4140, 4026, 4110, 4014] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4152, 4090, 4088, 4148, 4140, 4026, 4110, 4014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234667872 ingress_bytes=1643839488 link_bytes=1643839488 token_bytes=469221216 weight_movement_bytes=1409286144 total_nvlink_bytes=1878507360 weight_to_token_ratio=3.003458 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204814960e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114848 combine_payload_bytes=234766336 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=234881184 ingress_bytes=939409408 link_bytes=939409408 token_bytes=469647520 weight_movement_bytes=704643072 total_nvlink_bytes=1174290592 weight_to_token_ratio=1.500366 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.260300556e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235945984 dispatch_meta_bytes=115208 combine_payload_bytes=235945984 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236061192 ingress_bytes=2349875200 link_bytes=2349875200 token_bytes=472007176 weight_movement_bytes=2113929216 total_nvlink_bytes=2585936392 weight_to_token_ratio=4.478596 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315832023e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 2, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235405512 ingress_bytes=2349219840 link_bytes=2349219840 token_bytes=470696136 weight_movement_bytes=2113929216 total_nvlink_bytes=2584625352 weight_to_token_ratio=4.491070 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315465049e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32714 payload_rows_remote=28464 dup_rows=54 dup_rows_remote=44 dedup_ratio=0.001648 router_tokens_per_expert=[2098, 2038, 2083, 2035, 2092, 2033, 2037, 1968] payload_rows_per_dst=[4250, 4394, 4166, 4072, 3854, 4014, 4028, 3936] dup_rows_per_dst=[10, 44, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4250, 4394, 4166, 4072, 3854, 4014, 4028, 3936] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234438656 dispatch_meta_bytes=114472 combine_payload_bytes=234438656 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234553128 ingress_bytes=1643724800 link_bytes=1643724800 token_bytes=468991784 weight_movement_bytes=1409286144 total_nvlink_bytes=1878277928 weight_to_token_ratio=3.004927 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204172756e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=234880968 ingress_bytes=5167267840 link_bytes=5167267840 token_bytes=469647304 weight_movement_bytes=4932501504 total_nvlink_bytes=5402148808 weight_to_token_ratio=10.502565 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893454298e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2038, 2025, 2008, 2017, 2111, 2100, 2040] payload_rows_per_dst=[4090, 4076, 4050, 4016, 4034, 4222, 4200, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4076, 4050, 4016, 4034, 4222, 4200, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233177088 dispatch_meta_bytes=114032 combine_payload_bytes=233177088 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233291120 ingress_bytes=4461035520 link_bytes=4461035520 token_bytes=466468208 weight_movement_bytes=4227858432 total_nvlink_bytes=4694326640 weight_to_token_ratio=9.063551 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.497993678e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2075, 2034, 2033, 2062, 2040, 2023, 2071, 2046] payload_rows_per_dst=[4150, 4068, 4066, 4124, 4080, 4046, 4142, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4150, 4068, 4066, 4124, 4080, 4046, 4142, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=28574 dup_rows=92 dup_rows_remote=72 dedup_ratio=0.002808 router_tokens_per_expert=[2161, 2115, 1993, 2021, 2010, 2011, 2053, 2020] payload_rows_per_dst=[3956, 4230, 3832, 4312, 4020, 4020, 4204, 4102] dup_rows_per_dst=[0, 0, 0, 44, 0, 8, 20, 20] combine_rows_per_src=[3956, 4230, 3832, 4312, 4020, 4020, 4204, 4102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28770 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 1999, 2114, 2052, 1996, 2097, 2024, 2060] payload_rows_per_dst=[4084, 3998, 4228, 4104, 3992, 4194, 4048, 4120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 3998, 4228, 4104, 3992, 4194, 4048, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235503864 ingress_bytes=1644675072 link_bytes=1644675072 token_bytes=470892792 weight_movement_bytes=1409286144 total_nvlink_bytes=1880178936 weight_to_token_ratio=2.992796 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209493882e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2012, 2068, 2101, 2010, 2133, 2018, 2008] payload_rows_per_dst=[4068, 4024, 4136, 4202, 4020, 4266, 4036, 4016] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4024, 4136, 4202, 4020, 4266, 4036, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234766224 ingress_bytes=1643937792 link_bytes=1643937792 token_bytes=469417872 weight_movement_bytes=1409286144 total_nvlink_bytes=1878704016 weight_to_token_ratio=3.002200 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205365422e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2068, 2028, 2045, 2000, 2043, 2049, 2008, 2143] payload_rows_per_dst=[4136, 4056, 4090, 4000, 4086, 4098, 4016, 4286] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4136, 4056, 4090, 4000, 4086, 4098, 4016, 4286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 1968, 2042, 2008, 2078, 2125, 2036, 2053] payload_rows_per_dst=[4148, 3936, 4084, 4016, 4156, 4250, 4072, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 3936, 4084, 4016, 4156, 4250, 4072, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234078208 dispatch_meta_bytes=114584 combine_payload_bytes=234078208 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234192792 ingress_bytes=2348007424 link_bytes=2348007424 token_bytes=468271000 weight_movement_bytes=2113929216 total_nvlink_bytes=2582200216 weight_to_token_ratio=4.514329 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314786147e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32642 payload_rows_remote=28200 dup_rows=126 dup_rows_remote=70 dedup_ratio=0.003845 router_tokens_per_expert=[2074, 2086, 1992, 1988, 2014, 2071, 2114, 2045] payload_rows_per_dst=[4442, 4226, 3934, 4092, 4198, 4102, 3874, 3774] dup_rows_per_dst=[56, 10, 0, 26, 34, 0, 0, 0] combine_rows_per_src=[4442, 4226, 3934, 4092, 4198, 4102, 3874, 3774] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235683840 dispatch_meta_bytes=115080 combine_payload_bytes=235683840 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235798920 ingress_bytes=1644969984 link_bytes=1644969984 token_bytes=471482760 weight_movement_bytes=1409286144 total_nvlink_bytes=1880768904 weight_to_token_ratio=2.989051 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211145266e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234667872 ingress_bytes=3053125632 link_bytes=3053125632 token_bytes=469221216 weight_movement_bytes=2818572288 total_nvlink_bytes=3287793504 weight_to_token_ratio=6.006916 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709622910e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234979320 ingress_bytes=4462723072 link_bytes=4462723072 token_bytes=469843960 weight_movement_bytes=4227858432 total_nvlink_bytes=4697702392 weight_to_token_ratio=8.998431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498938637e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28628 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2068, 2063, 2038, 2070, 2034, 2029, 2027] payload_rows_per_dst=[4110, 4136, 4126, 4076, 4140, 4068, 4058, 4054] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4136, 4126, 4076, 4140, 4068, 4058, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235192416 ingress_bytes=3053649920 link_bytes=3053649920 token_bytes=470270048 weight_movement_bytes=2818572288 total_nvlink_bytes=3288842336 weight_to_token_ratio=5.993519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709916490e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=231014400 dispatch_meta_bytes=113080 combine_payload_bytes=231014400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231127480 ingress_bytes=1640300544 link_bytes=1640300544 token_bytes=462141880 weight_movement_bytes=1409286144 total_nvlink_bytes=1871428024 weight_to_token_ratio=3.049466 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.184998351e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28702 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2001, 2003, 2033, 2118, 2086, 2036, 2063] payload_rows_per_dst=[4088, 4002, 4006, 4066, 4236, 4172, 4072, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4002, 4006, 4066, 4236, 4172, 4072, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32582 payload_rows_remote=28214 dup_rows=186 dup_rows_remote=152 dedup_ratio=0.005676 router_tokens_per_expert=[2123, 2073, 2015, 1982, 2071, 2072, 2024, 2024] payload_rows_per_dst=[3668, 3726, 3964, 4108, 4142, 4208, 4398, 4368] dup_rows_per_dst=[0, 0, 30, 30, 0, 22, 70, 34] combine_rows_per_src=[3668, 3726, 3964, 4108, 4142, 4208, 4398, 4368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234520576 dispatch_meta_bytes=114512 combine_payload_bytes=234520576 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234635088 ingress_bytes=3053092864 link_bytes=3053092864 token_bytes=469155664 weight_movement_bytes=2818572288 total_nvlink_bytes=3287727952 weight_to_token_ratio=6.007755 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709604561e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2100, 2046, 2042, 2119, 2071, 2032, 1976, 1998] payload_rows_per_dst=[4200, 4092, 4084, 4238, 4142, 4064, 3952, 3996] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4200, 4092, 4084, 4238, 4142, 4064, 3952, 3996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235126784 dispatch_meta_bytes=114808 combine_payload_bytes=235126784 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235241592 ingress_bytes=2349056000 link_bytes=2349056000 token_bytes=470368376 weight_movement_bytes=2113929216 total_nvlink_bytes=2584297592 weight_to_token_ratio=4.494199 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315373306e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2044, 2101, 1996, 2040, 2058, 2063, 2033] payload_rows_per_dst=[4098, 4088, 4202, 3992, 4080, 4116, 4126, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4088, 4202, 3992, 4080, 4116, 4126, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=231129088 dispatch_meta_bytes=113464 combine_payload_bytes=231129088 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231242552 ingress_bytes=1640415232 link_bytes=1640415232 token_bytes=462371640 weight_movement_bytes=1409286144 total_nvlink_bytes=1871657784 weight_to_token_ratio=3.047951 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.185640556e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2092, 2056, 2041, 2042, 2041, 2034, 1993, 2085] payload_rows_per_dst=[4184, 4112, 4082, 4084, 4082, 4068, 3986, 4170] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4184, 4112, 4082, 4084, 4082, 4068, 3986, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28690 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2087, 2054, 2051, 2046, 2018, 2039, 2009] payload_rows_per_dst=[4160, 4174, 4108, 4102, 4092, 4036, 4078, 4018] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4174, 4108, 4102, 4092, 4036, 4078, 4018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32686 payload_rows_remote=28292 dup_rows=82 dup_rows_remote=28 dedup_ratio=0.002502 router_tokens_per_expert=[2034, 2110, 2074, 2019, 2054, 2049, 1987, 2057] payload_rows_per_dst=[4394, 3878, 4266, 4076, 4054, 4080, 3974, 3964] dup_rows_per_dst=[54, 0, 18, 10, 0, 0, 0, 0] combine_rows_per_src=[4394, 3878, 4266, 4076, 4054, 4080, 3974, 3964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234913792 dispatch_meta_bytes=114704 combine_payload_bytes=234913792 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235028496 ingress_bytes=2348843008 link_bytes=2348843008 token_bytes=469942288 weight_movement_bytes=2113929216 total_nvlink_bytes=2583871504 weight_to_token_ratio=4.498274 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315254039e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2014, 2039, 2074, 2071, 2101, 2035, 2041, 2009] payload_rows_per_dst=[4028, 4078, 4148, 4142, 4202, 4070, 4082, 4018] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4028, 4078, 4148, 4142, 4202, 4070, 4082, 4018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234126936 ingress_bytes=1643298816 link_bytes=1643298816 token_bytes=468139608 weight_movement_bytes=1409286144 total_nvlink_bytes=1877425752 weight_to_token_ratio=3.010397 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201787423e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235225200 ingress_bytes=2349039616 link_bytes=2349039616 token_bytes=470335600 weight_movement_bytes=2113929216 total_nvlink_bytes=2584264816 weight_to_token_ratio=4.494512 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315364131e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235028480 dispatch_meta_bytes=114760 combine_payload_bytes=235028480 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235143240 ingress_bytes=2348957696 link_bytes=2348957696 token_bytes=470171720 weight_movement_bytes=2113929216 total_nvlink_bytes=2584100936 weight_to_token_ratio=4.496079 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315318259e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2052, 2045, 2003, 2094, 2002, 2033, 2126, 2029] payload_rows_per_dst=[4104, 4090, 4006, 4188, 4004, 4066, 4252, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4104, 4090, 4006, 4188, 4004, 4066, 4252, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=231768064 dispatch_meta_bytes=113280 combine_payload_bytes=231768064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231881344 ingress_bytes=1641054208 link_bytes=1641054208 token_bytes=463649408 weight_movement_bytes=1409286144 total_nvlink_bytes=1872935552 weight_to_token_ratio=3.039551 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.189218555e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234126936 ingress_bytes=2347941888 link_bytes=2347941888 token_bytes=468139608 weight_movement_bytes=2113929216 total_nvlink_bytes=2582068824 weight_to_token_ratio=4.515596 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314749449e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32714 payload_rows_remote=28384 dup_rows=54 dup_rows_remote=6 dedup_ratio=0.001648 router_tokens_per_expert=[2039, 2041, 2068, 2024, 1986, 2057, 2145, 2024] payload_rows_per_dst=[4078, 3852, 4136, 3948, 3886, 4114, 4370, 4330] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 6, 48] combine_rows_per_src=[4078, 3852, 4136, 3948, 3886, 4114, 4370, 4330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234127360 dispatch_meta_bytes=114320 combine_payload_bytes=234127360 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234241680 ingress_bytes=2348056576 link_bytes=2348056576 token_bytes=468369040 weight_movement_bytes=2113929216 total_nvlink_bytes=2582298256 weight_to_token_ratio=4.513384 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314813670e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28806 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1947, 1981, 2043, 2058, 2094, 2088, 2045, 2128] payload_rows_per_dst=[3894, 3962, 4086, 4116, 4188, 4176, 4090, 4256] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3894, 3962, 4086, 4116, 4188, 4176, 4090, 4256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28616 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2015, 2137, 2076, 2014, 2073, 2055, 2009, 2005] payload_rows_per_dst=[4030, 4274, 4152, 4028, 4146, 4110, 4018, 4010] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4030, 4274, 4152, 4028, 4146, 4110, 4018, 4010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=232521728 dispatch_meta_bytes=113560 combine_payload_bytes=232521728 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232635288 ingress_bytes=2346450944 link_bytes=2346450944 token_bytes=465157016 weight_movement_bytes=2113929216 total_nvlink_bytes=2579086232 weight_to_token_ratio=4.544550 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313914583e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2085, 2037, 2011, 2035, 2060, 2032, 2073] payload_rows_per_dst=[4102, 4170, 4074, 4022, 4070, 4120, 4064, 4146] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4170, 4074, 4022, 4070, 4120, 4064, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2017, 2053, 2116, 2103, 2020, 2005, 2006, 2064] payload_rows_per_dst=[4034, 4106, 4232, 4206, 4040, 4010, 4012, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4034, 4106, 4232, 4206, 4040, 4010, 4012, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32612 payload_rows_remote=28330 dup_rows=156 dup_rows_remote=140 dedup_ratio=0.004761 router_tokens_per_expert=[2087, 1969, 2090, 1996, 2046, 2068, 2081, 2047] payload_rows_per_dst=[4282, 4218, 4268, 3878, 4272, 4324, 3852, 3518] dup_rows_per_dst=[16, 52, 6, 4, 40, 38, 0, 0] combine_rows_per_src=[4282, 4218, 4268, 3878, 4272, 4324, 3852, 3518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28678 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2028, 2099, 2082, 2072, 2045, 2037, 2002, 2019] payload_rows_per_dst=[4056, 4198, 4164, 4144, 4090, 4074, 4004, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4056, 4198, 4164, 4144, 4090, 4074, 4004, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235978752 dispatch_meta_bytes=115224 combine_payload_bytes=235978752 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236093976 ingress_bytes=3054551040 link_bytes=3054551040 token_bytes=472072728 weight_movement_bytes=2818572288 total_nvlink_bytes=3290645016 weight_to_token_ratio=5.970631 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710421079e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234422272 dispatch_meta_bytes=114464 combine_payload_bytes=234422272 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234536736 ingress_bytes=3052994560 link_bytes=3052994560 token_bytes=468959008 weight_movement_bytes=2818572288 total_nvlink_bytes=3287531296 weight_to_token_ratio=6.010274 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709549515e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28608 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1967, 2124, 2048, 2080, 2011, 2032, 2074, 2048] payload_rows_per_dst=[3934, 4248, 4096, 4160, 4022, 4064, 4148, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3934, 4248, 4096, 4160, 4022, 4064, 4148, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234799008 ingress_bytes=3053256704 link_bytes=3053256704 token_bytes=469483424 weight_movement_bytes=2818572288 total_nvlink_bytes=3288055712 weight_to_token_ratio=6.003561 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709696305e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235684176 ingress_bytes=3758784512 link_bytes=3758784512 token_bytes=471253328 weight_movement_bytes=3523215360 total_nvlink_bytes=3994468688 weight_to_token_ratio=7.476266 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104762427e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234930176 dispatch_meta_bytes=114712 combine_payload_bytes=234930176 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235044888 ingress_bytes=2348859392 link_bytes=2348859392 token_bytes=469975064 weight_movement_bytes=2113929216 total_nvlink_bytes=2583904280 weight_to_token_ratio=4.497960 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315263213e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=232079360 dispatch_meta_bytes=113880 combine_payload_bytes=232079360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232193240 ingress_bytes=1641365504 link_bytes=1641365504 token_bytes=464272600 weight_movement_bytes=1409286144 total_nvlink_bytes=1873558744 weight_to_token_ratio=3.035471 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.190961683e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234356736 dispatch_meta_bytes=114432 combine_payload_bytes=234356736 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234471168 ingress_bytes=2348285952 link_bytes=2348285952 token_bytes=468827904 weight_movement_bytes=2113929216 total_nvlink_bytes=2582757120 weight_to_token_ratio=4.508966 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314942111e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28516 dup_rows=84 dup_rows_remote=68 dedup_ratio=0.002563 router_tokens_per_expert=[2026, 2055, 2016, 1995, 2084, 2120, 2058, 2030] payload_rows_per_dst=[4052, 3956, 3976, 3920, 4214, 3976, 4422, 4168] dup_rows_per_dst=[0, 0, 0, 0, 16, 0, 52, 16] combine_rows_per_src=[4052, 3956, 3976, 3920, 4214, 3976, 4422, 4168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28678 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1971, 2045, 2074, 2046, 2060, 2056, 2038, 2094] payload_rows_per_dst=[3942, 4090, 4148, 4092, 4120, 4112, 4076, 4188] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3942, 4090, 4148, 4092, 4120, 4112, 4076, 4188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28844 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2096, 2113, 1962, 2049, 1994, 2086, 2033, 2051] payload_rows_per_dst=[4192, 4226, 3924, 4098, 3988, 4172, 4066, 4102] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4192, 4226, 3924, 4098, 3988, 4172, 4066, 4102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233603072 dispatch_meta_bytes=114336 combine_payload_bytes=233603072 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233717408 ingress_bytes=2347532288 link_bytes=2347532288 token_bytes=467320480 weight_movement_bytes=2113929216 total_nvlink_bytes=2581249696 weight_to_token_ratio=4.523511 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314520090e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28664 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 2049, 2026, 2071, 2090, 2052, 2038, 2011] payload_rows_per_dst=[4094, 4098, 4052, 4142, 4180, 4104, 4076, 4022] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 4098, 4052, 4142, 4180, 4104, 4076, 4022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28818 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2025, 2037, 2062, 2063, 2042, 2093, 1975, 2087] payload_rows_per_dst=[4050, 4074, 4124, 4126, 4084, 4186, 3950, 4174] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4050, 4074, 4124, 4126, 4084, 4186, 3950, 4174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1982, 2087, 2078, 2018, 2082, 2009, 2049, 2079] payload_rows_per_dst=[3964, 4174, 4156, 4036, 4164, 4018, 4098, 4158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3964, 4174, 4156, 4036, 4164, 4018, 4098, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28488 dup_rows=62 dup_rows_remote=50 dedup_ratio=0.001892 router_tokens_per_expert=[2097, 2095, 1986, 2040, 2033, 2070, 2060, 2003] payload_rows_per_dst=[4218, 4238, 3938, 4288, 4122, 4020, 3964, 3918] dup_rows_per_dst=[12, 16, 0, 24, 10, 0, 0, 0] combine_rows_per_src=[4218, 4238, 3938, 4288, 4122, 4020, 3964, 3918] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=236290048 dispatch_meta_bytes=115376 combine_payload_bytes=236290048 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236405424 ingress_bytes=3054862336 link_bytes=3054862336 token_bytes=472695472 weight_movement_bytes=2818572288 total_nvlink_bytes=3291267760 weight_to_token_ratio=5.962766 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710595392e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234930176 dispatch_meta_bytes=114712 combine_payload_bytes=234930176 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=235044888 ingress_bytes=4462788608 link_bytes=4462788608 token_bytes=469975064 weight_movement_bytes=4227858432 total_nvlink_bytes=4697833496 weight_to_token_ratio=8.995921 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498975335e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28720 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1990, 2042, 1989, 2024, 2064, 2051, 2082, 2142] payload_rows_per_dst=[3980, 4084, 3978, 4048, 4128, 4102, 4164, 4284] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3980, 4084, 3978, 4048, 4128, 4102, 4164, 4284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234815488 dispatch_meta_bytes=114656 combine_payload_bytes=234815488 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234930144 ingress_bytes=2348744704 link_bytes=2348744704 token_bytes=469745632 weight_movement_bytes=2113929216 total_nvlink_bytes=2583674848 weight_to_token_ratio=4.500157 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315198993e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=236077056 dispatch_meta_bytes=115272 combine_payload_bytes=236077056 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236192328 ingress_bytes=1645363200 link_bytes=1645363200 token_bytes=472269384 weight_movement_bytes=1409286144 total_nvlink_bytes=1881555528 weight_to_token_ratio=2.984073 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213347112e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234438384 ingress_bytes=2348253184 link_bytes=2348253184 token_bytes=468762352 weight_movement_bytes=2113929216 total_nvlink_bytes=2582691568 weight_to_token_ratio=4.509597 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314923762e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233373696 dispatch_meta_bytes=114152 combine_payload_bytes=233373696 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233487848 ingress_bytes=2347302912 link_bytes=2347302912 token_bytes=466861544 weight_movement_bytes=2113929216 total_nvlink_bytes=2580790760 weight_to_token_ratio=4.527957 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314391649e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=28428 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.001465 router_tokens_per_expert=[2056, 2017, 2042, 2045, 2010, 2002, 2072, 2140] payload_rows_per_dst=[4112, 4034, 3978, 4146, 3926, 4074, 4158, 4292] dup_rows_per_dst=[0, 0, 0, 22, 0, 24, 2, 0] combine_rows_per_src=[4112, 4034, 3978, 4146, 3926, 4074, 4158, 4292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235274240 dispatch_meta_bytes=114880 combine_payload_bytes=235274240 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235389120 ingress_bytes=3053846528 link_bytes=3053846528 token_bytes=470663360 weight_movement_bytes=2818572288 total_nvlink_bytes=3289235648 weight_to_token_ratio=5.988510 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710026582e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2032, 2005, 2023, 2094, 2067, 2037, 2058, 2068] payload_rows_per_dst=[4064, 4010, 4046, 4188, 4134, 4074, 4116, 4136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4064, 4010, 4046, 4188, 4134, 4074, 4116, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2019, 2065, 2017, 2129, 2024, 2026, 2087, 2017] payload_rows_per_dst=[4038, 4130, 4034, 4258, 4048, 4052, 4174, 4034] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4038, 4130, 4034, 4258, 4048, 4052, 4174, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=232882176 dispatch_meta_bytes=113904 combine_payload_bytes=232882176 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=232996080 ingress_bytes=4460740608 link_bytes=4460740608 token_bytes=465878256 weight_movement_bytes=4227858432 total_nvlink_bytes=4693736688 weight_to_token_ratio=9.075028 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.497828540e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235405512 ingress_bytes=2349219840 link_bytes=2349219840 token_bytes=470696136 weight_movement_bytes=2113929216 total_nvlink_bytes=2584625352 weight_to_token_ratio=4.491070 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315465049e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28628 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2084, 1989, 2091, 2081, 1982, 2023, 2070, 2064] payload_rows_per_dst=[4168, 3978, 4182, 4162, 3964, 4046, 4140, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4168, 3978, 4182, 4162, 3964, 4046, 4140, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2072, 2044, 2060, 2013, 2046, 2008, 2032, 2109] payload_rows_per_dst=[4144, 4088, 4120, 4026, 4092, 4016, 4064, 4218] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4144, 4088, 4120, 4026, 4092, 4016, 4064, 4218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28626 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2036, 1995, 2077, 1982, 2071, 2085, 2120, 2018] payload_rows_per_dst=[4072, 3990, 4154, 3964, 4142, 4170, 4240, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4072, 3990, 4154, 3964, 4142, 4170, 4240, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234717048 ingress_bytes=1643888640 link_bytes=1643888640 token_bytes=469319544 weight_movement_bytes=1409286144 total_nvlink_bytes=1878605688 weight_to_token_ratio=3.002829 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205090191e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2110, 2059, 1994, 2018, 2017, 2112, 2031, 2043] payload_rows_per_dst=[4220, 4118, 3988, 4036, 4034, 4224, 4062, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4220, 4118, 3988, 4036, 4034, 4224, 4062, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32740 payload_rows_remote=28598 dup_rows=28 dup_rows_remote=24 dedup_ratio=0.000854 router_tokens_per_expert=[2038, 2118, 2107, 2042, 2064, 2006, 2000, 2009] payload_rows_per_dst=[4142, 4412, 4108, 4090, 4096, 3874, 4000, 4018] dup_rows_per_dst=[4, 24, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4412, 4108, 4090, 4096, 3874, 4000, 4018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234520576 dispatch_meta_bytes=114512 combine_payload_bytes=234520576 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234635088 ingress_bytes=4462379008 link_bytes=4462379008 token_bytes=469155664 weight_movement_bytes=4227858432 total_nvlink_bytes=4697014096 weight_to_token_ratio=9.011633 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498745976e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235651392 ingress_bytes=1644822528 link_bytes=1644822528 token_bytes=471187776 weight_movement_bytes=1409286144 total_nvlink_bytes=1880473920 weight_to_token_ratio=2.990923 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210319574e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28474 dup_rows=62 dup_rows_remote=32 dedup_ratio=0.001892 router_tokens_per_expert=[2057, 2096, 2062, 2132, 1980, 2005, 2033, 2019] payload_rows_per_dst=[3902, 3968, 4308, 4264, 3960, 3974, 4098, 4232] dup_rows_per_dst=[0, 0, 28, 0, 0, 0, 4, 30] combine_rows_per_src=[3902, 3968, 4308, 4264, 3960, 3974, 4098, 4232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234504192 dispatch_meta_bytes=114504 combine_payload_bytes=234504192 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234618696 ingress_bytes=1643790336 link_bytes=1643790336 token_bytes=469122888 weight_movement_bytes=1409286144 total_nvlink_bytes=1878409032 weight_to_token_ratio=3.004087 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204539730e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235487472 ingress_bytes=1644658688 link_bytes=1644658688 token_bytes=470860016 weight_movement_bytes=1409286144 total_nvlink_bytes=1880146160 weight_to_token_ratio=2.993004 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209402138e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114488 combine_payload_bytes=234274816 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234389304 ingress_bytes=3052847104 link_bytes=3052847104 token_bytes=468664120 weight_movement_bytes=2818572288 total_nvlink_bytes=3287236408 weight_to_token_ratio=6.014056 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709466946e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2019, 2065, 2060, 2002, 2097, 2019, 2035, 2087] payload_rows_per_dst=[4038, 4130, 4120, 4004, 4194, 4038, 4070, 4174] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4038, 4130, 4120, 4004, 4194, 4038, 4070, 4174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233259008 dispatch_meta_bytes=114024 combine_payload_bytes=233259008 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233373032 ingress_bytes=3756474368 link_bytes=3756474368 token_bytes=466632040 weight_movement_bytes=3523215360 total_nvlink_bytes=3989847400 weight_to_token_ratio=7.550307 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103468843e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234799008 ingress_bytes=2348613632 link_bytes=2348613632 token_bytes=469483424 weight_movement_bytes=2113929216 total_nvlink_bytes=2583412640 weight_to_token_ratio=4.502671 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315125598e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28612 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2078, 2035, 2024, 2067, 2026, 2038, 2092] payload_rows_per_dst=[4048, 4156, 4070, 4048, 4134, 4052, 4076, 4184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4156, 4070, 4048, 4134, 4052, 4076, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28600 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2051, 2027, 2060, 2011, 2039, 2084, 2048] payload_rows_per_dst=[4128, 4102, 4054, 4120, 4022, 4078, 4168, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4102, 4054, 4120, 4022, 4078, 4168, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28556 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2025, 2054, 2010, 2075, 2106, 2043, 2006] payload_rows_per_dst=[4130, 4050, 4108, 4020, 4150, 4212, 4086, 4012] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4050, 4108, 4020, 4150, 4212, 4086, 4012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2107, 1957, 2058, 2068, 2092, 1988, 2085, 2029] payload_rows_per_dst=[4214, 3914, 4116, 4136, 4184, 3976, 4170, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4214, 3914, 4116, 4136, 4184, 3976, 4170, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28674 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2085, 2033, 2062, 2047, 2081, 1978, 2048, 2050] payload_rows_per_dst=[4170, 4066, 4124, 4094, 4162, 3956, 4096, 4100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4170, 4066, 4124, 4094, 4162, 3956, 4096, 4100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32722 payload_rows_remote=28458 dup_rows=46 dup_rows_remote=2 dedup_ratio=0.001404 router_tokens_per_expert=[2043, 1982, 2067, 2050, 2040, 2068, 2068, 2066] payload_rows_per_dst=[4264, 3976, 4098, 4100, 4080, 4136, 3942, 4126] dup_rows_per_dst=[44, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4264, 3976, 4098, 4100, 4080, 4136, 3942, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234389504 dispatch_meta_bytes=114448 combine_payload_bytes=234389504 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234503952 ingress_bytes=2348318720 link_bytes=2348318720 token_bytes=468893456 weight_movement_bytes=2113929216 total_nvlink_bytes=2582822672 weight_to_token_ratio=4.508336 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314960460e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234291200 dispatch_meta_bytes=114400 combine_payload_bytes=234291200 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234405600 ingress_bytes=2348220416 link_bytes=2348220416 token_bytes=468696800 weight_movement_bytes=2113929216 total_nvlink_bytes=2582626016 weight_to_token_ratio=4.510228 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314905413e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32630 payload_rows_remote=28326 dup_rows=138 dup_rows_remote=86 dedup_ratio=0.004211 router_tokens_per_expert=[2067, 2125, 2047, 2033, 2047, 2046, 1993, 2026] payload_rows_per_dst=[3830, 3950, 3824, 3948, 4094, 4228, 4452, 4304] dup_rows_per_dst=[0, 0, 0, 6, 0, 32, 48, 52] combine_rows_per_src=[3830, 3950, 3824, 3948, 4094, 4228, 4452, 4304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114224 combine_payload_bytes=233930752 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=234044976 ingress_bytes=3757146112 link_bytes=3757146112 token_bytes=467975728 weight_movement_bytes=3523215360 total_nvlink_bytes=3991191088 weight_to_token_ratio=7.528628 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103844992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234160128 dispatch_meta_bytes=114336 combine_payload_bytes=234160128 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234274464 ingress_bytes=3052732416 link_bytes=3052732416 token_bytes=468434592 weight_movement_bytes=2818572288 total_nvlink_bytes=3287006880 weight_to_token_ratio=6.017003 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709402726e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234897408 dispatch_meta_bytes=114696 combine_payload_bytes=234897408 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235012104 ingress_bytes=2348826624 link_bytes=2348826624 token_bytes=469909512 weight_movement_bytes=2113929216 total_nvlink_bytes=2583838728 weight_to_token_ratio=4.498588 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315244865e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233127936 dispatch_meta_bytes=113840 combine_payload_bytes=233127936 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233241776 ingress_bytes=3756343296 link_bytes=3756343296 token_bytes=466369712 weight_movement_bytes=3523215360 total_nvlink_bytes=3989585072 weight_to_token_ratio=7.554554 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103395448e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2090, 2059, 2057, 2006, 2102, 1976, 2056] payload_rows_per_dst=[4076, 4180, 4118, 4114, 4012, 4204, 3952, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4180, 4118, 4114, 4012, 4204, 3952, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=232046592 dispatch_meta_bytes=113648 combine_payload_bytes=232046592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232160240 ingress_bytes=1641332736 link_bytes=1641332736 token_bytes=464206832 weight_movement_bytes=1409286144 total_nvlink_bytes=1873492976 weight_to_token_ratio=3.035901 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.190778196e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234700800 dispatch_meta_bytes=114600 combine_payload_bytes=234700800 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234815400 ingress_bytes=4462559232 link_bytes=4462559232 token_bytes=469516200 weight_movement_bytes=4227858432 total_nvlink_bytes=4697374632 weight_to_token_ratio=9.004713 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498846894e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2052, 2042, 2118, 2023, 2062, 2094, 1999, 1994] payload_rows_per_dst=[4104, 4084, 4236, 4046, 4124, 4188, 3998, 3988] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4104, 4084, 4236, 4046, 4124, 4188, 3998, 3988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28738 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2003, 2067, 2025, 2141, 2008, 2042, 2015, 2083] payload_rows_per_dst=[4006, 4134, 4050, 4282, 4016, 4084, 4030, 4166] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4006, 4134, 4050, 4282, 4016, 4084, 4030, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28542 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2111, 2002, 2096, 2026, 2040, 2113, 2011, 1985] payload_rows_per_dst=[4222, 4004, 4192, 4052, 4080, 4226, 4022, 3970] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4222, 4004, 4192, 4052, 4080, 4226, 4022, 3970] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2030, 2044, 2064, 2095, 2036, 2042, 2018] payload_rows_per_dst=[4110, 4060, 4088, 4128, 4190, 4072, 4084, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4060, 4088, 4128, 4190, 4072, 4084, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32698 payload_rows_remote=28528 dup_rows=70 dup_rows_remote=66 dedup_ratio=0.002136 router_tokens_per_expert=[2078, 2069, 2056, 2076, 2035, 1955, 2029, 2086] payload_rows_per_dst=[4170, 4160, 4168, 4394, 4056, 4000, 4058, 3692] dup_rows_per_dst=[4, 2, 10, 44, 0, 10, 0, 0] combine_rows_per_src=[4170, 4160, 4168, 4394, 4056, 4000, 4058, 3692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2127, 2115, 2004, 2093, 1993, 2004, 2031, 2017] payload_rows_per_dst=[4254, 4230, 4008, 4186, 3986, 4008, 4062, 4034] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4254, 4230, 4008, 4186, 3986, 4008, 4062, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114736 combine_payload_bytes=234979328 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235094064 ingress_bytes=2348908544 link_bytes=2348908544 token_bytes=470073392 weight_movement_bytes=2113929216 total_nvlink_bytes=2584002608 weight_to_token_ratio=4.497020 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315290736e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235421696 dispatch_meta_bytes=114952 combine_payload_bytes=235421696 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235536648 ingress_bytes=1644707840 link_bytes=1644707840 token_bytes=470958344 weight_movement_bytes=1409286144 total_nvlink_bytes=1880244488 weight_to_token_ratio=2.992380 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209677369e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32606 payload_rows_remote=28394 dup_rows=162 dup_rows_remote=152 dedup_ratio=0.004944 router_tokens_per_expert=[2089, 2005, 2062, 2059, 2084, 1983, 2073, 2029] payload_rows_per_dst=[3508, 4046, 3896, 3980, 4338, 4278, 4348, 4212] dup_rows_per_dst=[0, 20, 0, 10, 24, 52, 46, 10] combine_rows_per_src=[3508, 4046, 3896, 3980, 4338, 4278, 4348, 4212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114168 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233930232 ingress_bytes=3052388352 link_bytes=3052388352 token_bytes=467746296 weight_movement_bytes=2818572288 total_nvlink_bytes=3286318584 weight_to_token_ratio=6.025857 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709210064e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28628 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2112, 2054, 2070, 2019, 2055, 2001, 2040, 2033] payload_rows_per_dst=[4224, 4108, 4140, 4038, 4110, 4002, 4080, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4224, 4108, 4140, 4038, 4110, 4002, 4080, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114312 combine_payload_bytes=234110976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234225288 ingress_bytes=1643397120 link_bytes=1643397120 token_bytes=468336264 weight_movement_bytes=1409286144 total_nvlink_bytes=1877622408 weight_to_token_ratio=3.009133 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202337884e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233701376 dispatch_meta_bytes=114376 combine_payload_bytes=233701376 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233815752 ingress_bytes=3052273664 link_bytes=3052273664 token_bytes=467517128 weight_movement_bytes=2818572288 total_nvlink_bytes=3286089416 weight_to_token_ratio=6.028811 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709145844e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234143744 dispatch_meta_bytes=114328 combine_payload_bytes=234143744 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234258072 ingress_bytes=3052716032 link_bytes=3052716032 token_bytes=468401816 weight_movement_bytes=2818572288 total_nvlink_bytes=3286974104 weight_to_token_ratio=6.017424 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709393551e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232603648 dispatch_meta_bytes=114184 combine_payload_bytes=232603648 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=232717832 ingress_bytes=1641889792 link_bytes=1641889792 token_bytes=465321480 weight_movement_bytes=1409286144 total_nvlink_bytes=1874607624 weight_to_token_ratio=3.028629 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.193897477e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234520576 dispatch_meta_bytes=114512 combine_payload_bytes=234520576 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234635088 ingress_bytes=4462379008 link_bytes=4462379008 token_bytes=469155664 weight_movement_bytes=4227858432 total_nvlink_bytes=4697014096 weight_to_token_ratio=9.011633 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498745976e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28744 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2079, 2012, 2065, 2096, 2013, 2050, 2056, 2013] payload_rows_per_dst=[4158, 4024, 4130, 4192, 4026, 4100, 4112, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4158, 4024, 4130, 4192, 4026, 4100, 4112, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2039, 2087, 2157, 1995, 1994, 2074, 2006, 2032] payload_rows_per_dst=[4078, 4174, 4314, 3990, 3988, 4148, 4012, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4078, 4174, 4314, 3990, 3988, 4148, 4012, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28796 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2064, 2047, 2072, 2013, 1986, 2041, 2097] payload_rows_per_dst=[4128, 4128, 4094, 4144, 4026, 3972, 4082, 4194] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4128, 4094, 4144, 4026, 3972, 4082, 4194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28656 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2024, 2053, 2012, 2056, 2041, 2088, 2044] payload_rows_per_dst=[4132, 4048, 4106, 4024, 4112, 4082, 4176, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4048, 4106, 4024, 4112, 4082, 4176, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235470848 dispatch_meta_bytes=114976 combine_payload_bytes=235470848 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235585824 ingress_bytes=3054043136 link_bytes=3054043136 token_bytes=471056672 weight_movement_bytes=2818572288 total_nvlink_bytes=3289628960 weight_to_token_ratio=5.983510 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710136674e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2054, 2122, 2000, 2005, 2066, 2050, 2021] payload_rows_per_dst=[4132, 4108, 4244, 4000, 4010, 4132, 4100, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4108, 4244, 4000, 4010, 4132, 4100, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32664 payload_rows_remote=28330 dup_rows=104 dup_rows_remote=68 dedup_ratio=0.003174 router_tokens_per_expert=[1979, 2029, 2066, 2066, 2087, 2054, 2058, 2045] payload_rows_per_dst=[4334, 4086, 4132, 4412, 3808, 4010, 4042, 3840] dup_rows_per_dst=[36, 4, 0, 56, 0, 8, 0, 0] combine_rows_per_src=[4334, 4086, 4132, 4412, 3808, 4010, 4042, 3840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32622 payload_rows_remote=28212 dup_rows=146 dup_rows_remote=128 dedup_ratio=0.004456 router_tokens_per_expert=[1996, 2032, 2082, 2099, 2029, 2043, 1979, 2124] payload_rows_per_dst=[3924, 4122, 3592, 4064, 4078, 4106, 4326, 4410] dup_rows_per_dst=[0, 2, 0, 52, 10, 4, 60, 18] combine_rows_per_src=[3924, 4122, 3592, 4064, 4078, 4106, 4326, 4410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235684176 ingress_bytes=2349498368 link_bytes=2349498368 token_bytes=471253328 weight_movement_bytes=2113929216 total_nvlink_bytes=2585182544 weight_to_token_ratio=4.485760 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315621013e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28724 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2015, 2021, 2022, 2104, 2105, 2047, 1986, 2084] payload_rows_per_dst=[4030, 4042, 4044, 4208, 4210, 4094, 3972, 4168] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4030, 4042, 4044, 4208, 4210, 4094, 3972, 4168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235896832 dispatch_meta_bytes=115184 combine_payload_bytes=235896832 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236012016 ingress_bytes=1645182976 link_bytes=1645182976 token_bytes=471908848 weight_movement_bytes=1409286144 total_nvlink_bytes=1881194992 weight_to_token_ratio=2.986352 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212337932e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234749952 dispatch_meta_bytes=114624 combine_payload_bytes=234749952 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234864576 ingress_bytes=2348679168 link_bytes=2348679168 token_bytes=469614528 weight_movement_bytes=2113929216 total_nvlink_bytes=2583543744 weight_to_token_ratio=4.501414 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315162295e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235667456 dispatch_meta_bytes=115072 combine_payload_bytes=235667456 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235782528 ingress_bytes=2349596672 link_bytes=2349596672 token_bytes=471449984 weight_movement_bytes=2113929216 total_nvlink_bytes=2585379200 weight_to_token_ratio=4.483889 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315676059e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=232079360 dispatch_meta_bytes=113592 combine_payload_bytes=232079360 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232192952 ingress_bytes=1641365504 link_bytes=1641365504 token_bytes=464272312 weight_movement_bytes=1409286144 total_nvlink_bytes=1873558456 weight_to_token_ratio=3.035473 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.190961683e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=231112704 dispatch_meta_bytes=113360 combine_payload_bytes=231112704 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=231226064 ingress_bytes=3049684992 link_bytes=3049684992 token_bytes=462338768 weight_movement_bytes=2818572288 total_nvlink_bytes=3280911056 weight_to_token_ratio=6.096336 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.707696295e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235307008 dispatch_meta_bytes=114896 combine_payload_bytes=235307008 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=235421904 ingress_bytes=5167808512 link_bytes=5167808512 token_bytes=470728912 weight_movement_bytes=4932501504 total_nvlink_bytes=5403230416 weight_to_token_ratio=10.478433 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893757052e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2100, 2018, 2053, 1982, 2130, 2043, 2024, 2034] payload_rows_per_dst=[4200, 4036, 4106, 3964, 4260, 4086, 4048, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4200, 4036, 4106, 3964, 4260, 4086, 4048, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2105, 2081, 2139, 1961, 1968, 2040, 2009, 2081] payload_rows_per_dst=[4210, 4162, 4278, 3922, 3936, 4080, 4018, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4210, 4162, 4278, 3922, 3936, 4080, 4018, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1983, 2043, 2110, 2048, 2048, 2087, 2005, 2060] payload_rows_per_dst=[3966, 4086, 4220, 4096, 4096, 4174, 4010, 4120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3966, 4086, 4220, 4096, 4096, 4174, 4010, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235487472 ingress_bytes=1644658688 link_bytes=1644658688 token_bytes=470860016 weight_movement_bytes=1409286144 total_nvlink_bytes=1880146160 weight_to_token_ratio=2.993004 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209402138e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28542 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1991, 2068, 2077, 2018, 2113, 2044, 2050, 2023] payload_rows_per_dst=[3982, 4136, 4154, 4036, 4226, 4088, 4100, 4046] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3982, 4136, 4154, 4036, 4226, 4088, 4100, 4046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=28636 dup_rows=34 dup_rows_remote=30 dedup_ratio=0.001038 router_tokens_per_expert=[2031, 2071, 2051, 1954, 2040, 2139, 2059, 2039] payload_rows_per_dst=[4062, 4102, 4116, 3742, 4084, 4278, 4252, 4098] dup_rows_per_dst=[0, 0, 12, 0, 4, 0, 14, 4] combine_rows_per_src=[4062, 4102, 4116, 3742, 4084, 4278, 4252, 4098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2024, 2060, 2015, 2055, 2084, 2026, 2076] payload_rows_per_dst=[4088, 4048, 4120, 4030, 4110, 4168, 4052, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4048, 4120, 4030, 4110, 4168, 4052, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235635000 ingress_bytes=1644806144 link_bytes=1644806144 token_bytes=471155000 weight_movement_bytes=1409286144 total_nvlink_bytes=1880441144 weight_to_token_ratio=2.991131 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210227831e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234242048 dispatch_meta_bytes=114376 combine_payload_bytes=234242048 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234356424 ingress_bytes=2348171264 link_bytes=2348171264 token_bytes=468598472 weight_movement_bytes=2113929216 total_nvlink_bytes=2582527688 weight_to_token_ratio=4.511174 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314877890e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28462 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2108, 2013, 2153, 2046, 1976, 2043, 1994] payload_rows_per_dst=[4102, 4216, 4026, 4306, 4092, 3952, 4086, 3988] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4216, 4026, 4306, 4092, 3952, 4086, 3988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114168 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233930232 ingress_bytes=3757031424 link_bytes=3757031424 token_bytes=467746296 weight_movement_bytes=3523215360 total_nvlink_bytes=3990961656 weight_to_token_ratio=7.532321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103780771e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=28430 dup_rows=48 dup_rows_remote=16 dedup_ratio=0.001465 router_tokens_per_expert=[2027, 2028, 2067, 2021, 2098, 1949, 2118, 2076] payload_rows_per_dst=[4290, 4056, 3902, 4078, 4154, 3898, 4236, 4106] dup_rows_per_dst=[32, 0, 10, 6, 0, 0, 0, 0] combine_rows_per_src=[4290, 4056, 3902, 4078, 4154, 3898, 4236, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28778 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 1995, 2049, 2063, 2092, 1982, 2061, 2062] payload_rows_per_dst=[4160, 3990, 4098, 4126, 4184, 3964, 4122, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 3990, 4098, 4126, 4184, 3964, 4122, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234586112 dispatch_meta_bytes=114664 combine_payload_bytes=234586112 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234700776 ingress_bytes=3053158400 link_bytes=3053158400 token_bytes=469286888 weight_movement_bytes=2818572288 total_nvlink_bytes=3287859176 weight_to_token_ratio=6.006075 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709641259e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234799008 ingress_bytes=2348613632 link_bytes=2348613632 token_bytes=469483424 weight_movement_bytes=2113929216 total_nvlink_bytes=2583412640 weight_to_token_ratio=4.502671 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315125598e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233160704 dispatch_meta_bytes=113848 combine_payload_bytes=233160704 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233274552 ingress_bytes=2347089920 link_bytes=2347089920 token_bytes=466435256 weight_movement_bytes=2113929216 total_nvlink_bytes=2580364472 weight_to_token_ratio=4.532096 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314272383e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232898560 dispatch_meta_bytes=113784 combine_payload_bytes=232898560 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=233012344 ingress_bytes=1642184704 link_bytes=1642184704 token_bytes=465910904 weight_movement_bytes=1409286144 total_nvlink_bytes=1875197048 weight_to_token_ratio=3.024798 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195548861e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2026, 2078, 2024, 2076, 2046, 2001, 2117] payload_rows_per_dst=[4032, 4052, 4156, 4048, 4152, 4092, 4002, 4234] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4052, 4156, 4048, 4152, 4092, 4002, 4234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235749376 dispatch_meta_bytes=115112 combine_payload_bytes=235749376 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235864488 ingress_bytes=1645035520 link_bytes=1645035520 token_bytes=471613864 weight_movement_bytes=1409286144 total_nvlink_bytes=1880900008 weight_to_token_ratio=2.988220 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211512240e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28780 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2098, 2069, 2031, 1994, 2074, 2007, 2066] payload_rows_per_dst=[4090, 4196, 4138, 4062, 3988, 4148, 4014, 4132] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4196, 4138, 4062, 3988, 4148, 4014, 4132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1986, 2067, 1977, 2006, 2054, 2104, 2049, 2141] payload_rows_per_dst=[3972, 4134, 3954, 4012, 4108, 4208, 4098, 4282] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3972, 4134, 3954, 4012, 4108, 4208, 4098, 4282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7455855496 prefetch_bytes=21139292160 reduce_bytes=14797504512 weight_movement_bytes=35936796672 egress_bytes=3728839560 ingress_bytes=39663812608 link_bytes=39663812608 total_nvlink_bytes=43392652168 weight_to_token_ratio=4.819943 prefetch_share_of_ingress=0.532962 dedup_ratio=0.002480 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=21 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016098 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.388129963e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235651072 dispatch_meta_bytes=115064 combine_payload_bytes=235651072 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235766136 ingress_bytes=3054223360 link_bytes=3054223360 token_bytes=471417208 weight_movement_bytes=2818572288 total_nvlink_bytes=3289989496 weight_to_token_ratio=5.978934 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710237592e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32636 payload_rows_remote=28414 dup_rows=132 dup_rows_remote=110 dedup_ratio=0.004028 router_tokens_per_expert=[2028, 2113, 2120, 2046, 1987, 2028, 2021, 2041] payload_rows_per_dst=[4042, 3782, 4038, 4122, 4064, 4374, 3992, 4222] dup_rows_per_dst=[0, 0, 0, 4, 22, 62, 22, 22] combine_rows_per_src=[4042, 3782, 4038, 4122, 4064, 4374, 3992, 4222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235765760 dispatch_meta_bytes=115120 combine_payload_bytes=235765760 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235880880 ingress_bytes=2349694976 link_bytes=2349694976 token_bytes=471646640 weight_movement_bytes=2113929216 total_nvlink_bytes=2585575856 weight_to_token_ratio=4.482019 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315731105e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28736 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2102, 2016, 2009, 1960, 2105, 2025, 2113] payload_rows_per_dst=[4108, 4204, 4032, 4018, 3920, 4210, 4050, 4226] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4204, 4032, 4018, 3920, 4210, 4050, 4226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2001, 2082, 2103, 2048, 2092, 1958, 2068, 2032] payload_rows_per_dst=[4002, 4164, 4206, 4096, 4184, 3916, 4136, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4002, 4164, 4206, 4096, 4184, 3916, 4136, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114240 combine_payload_bytes=233963520 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234077760 ingress_bytes=2347892736 link_bytes=2347892736 token_bytes=468041280 weight_movement_bytes=2113929216 total_nvlink_bytes=2581970496 weight_to_token_ratio=4.516544 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314721926e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28510 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1977, 2129, 2086, 2036, 2054, 2062, 2052, 1988] payload_rows_per_dst=[3954, 4258, 4172, 4072, 4108, 4124, 4104, 3976] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3954, 4258, 4172, 4072, 4108, 4124, 4104, 3976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232767488 dispatch_meta_bytes=114096 combine_payload_bytes=232767488 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=232881584 ingress_bytes=1642053632 link_bytes=1642053632 token_bytes=465649072 weight_movement_bytes=1409286144 total_nvlink_bytes=1874935216 weight_to_token_ratio=3.026498 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.194814912e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7514324088 prefetch_bytes=19025362944 reduce_bytes=17616076800 weight_movement_bytes=36641439744 egress_bytes=3758079096 ingress_bytes=40397684736 link_bytes=40397684736 total_nvlink_bytes=44155763832 weight_to_token_ratio=4.876212 prefetch_share_of_ingress=0.470952 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=25 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.413813573e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235405312 dispatch_meta_bytes=114944 combine_payload_bytes=235405312 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235520256 ingress_bytes=3758620672 link_bytes=3758620672 token_bytes=470925568 weight_movement_bytes=3523215360 total_nvlink_bytes=3994140928 weight_to_token_ratio=7.481470 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104670684e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234995712 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=469876736 weight_movement_bytes=1409286144 total_nvlink_bytes=1879162880 weight_to_token_ratio=2.999268 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2081, 1933, 2077, 2085, 2119, 2062, 2049, 1978] payload_rows_per_dst=[4162, 3866, 4154, 4170, 4238, 4124, 4098, 3956] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4162, 3866, 4154, 4170, 4238, 4124, 4098, 3956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7453627120 prefetch_bytes=16911433728 reduce_bytes=18320719872 weight_movement_bytes=35232153600 egress_bytes=3727725296 ingress_bytes=38958055424 link_bytes=38958055424 total_nvlink_bytes=42685780720 weight_to_token_ratio=4.726847 prefetch_share_of_ingress=0.434093 dedup_ratio=0.002769 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=26 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.363430303e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233553920 dispatch_meta_bytes=114040 combine_payload_bytes=233553920 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=233667960 ingress_bytes=5166055424 link_bytes=5166055424 token_bytes=467221880 weight_movement_bytes=4932501504 total_nvlink_bytes=5399723384 weight_to_token_ratio=10.557086 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.892775396e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7522092000 prefetch_bytes=17616076800 reduce_bytes=26776436736 weight_movement_bytes=44392513536 egress_bytes=3761964000 ingress_bytes=48152641536 link_bytes=48152641536 total_nvlink_bytes=51914605536 weight_to_token_ratio=5.901618 prefetch_share_of_ingress=0.365838 dedup_ratio=0.000000 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=38 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.685216829e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7515209040 prefetch_bytes=21139292160 reduce_bytes=9160359936 weight_movement_bytes=30299652096 egress_bytes=3758521680 ingress_bytes=34056339456 link_bytes=34056339456 total_nvlink_bytes=37814861136 weight_to_token_ratio=4.031778 prefetch_share_of_ingress=0.620715 dedup_ratio=0.000000 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=13 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016098 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.191883032e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28856 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2121, 2058, 2070, 2045, 1956, 2058, 2032] payload_rows_per_dst=[4088, 4242, 4116, 4140, 4090, 3912, 4116, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4242, 4116, 4140, 4090, 3912, 4116, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234979320 ingress_bytes=2348793856 link_bytes=2348793856 token_bytes=469843960 weight_movement_bytes=2113929216 total_nvlink_bytes=2583773176 weight_to_token_ratio=4.499215 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315226516e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7527500040 prefetch_bytes=20434649088 reduce_bytes=16206790656 weight_movement_bytes=36641439744 egress_bytes=3764668680 ingress_bytes=40404271104 link_bytes=40404271104 total_nvlink_bytes=44168939784 weight_to_token_ratio=4.867677 prefetch_share_of_ingress=0.505755 dedup_ratio=0.000000 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=23 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.414044079e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=236388352 dispatch_meta_bytes=115424 combine_payload_bytes=236388352 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236503776 ingress_bytes=1645674496 link_bytes=1645674496 token_bytes=472892128 weight_movement_bytes=1409286144 total_nvlink_bytes=1882178272 weight_to_token_ratio=2.980143 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.215090239e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7528843856 prefetch_bytes=18320719872 reduce_bytes=23253221376 weight_movement_bytes=41573941248 egress_bytes=3765340752 ingress_bytes=45337444352 link_bytes=45337444352 total_nvlink_bytes=49102785104 weight_to_token_ratio=5.521956 prefetch_share_of_ingress=0.404097 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=33 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.586692272e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7512849168 prefetch_bytes=17616076800 reduce_bytes=26071793664 weight_movement_bytes=43687870464 egress_bytes=3757341456 ingress_bytes=47443378176 link_bytes=47443378176 total_nvlink_bytes=51200719632 weight_to_token_ratio=5.815087 prefetch_share_of_ingress=0.371307 dedup_ratio=0.000000 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=37 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.660394462e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none - [2026-09-16 21:35:53] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 578.5 | throughput per GPU (TFLOP/s/GPU): 576.3 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.023814E+01 | loss scale: 1.0 | grad norm: 6.270 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32634 payload_rows_remote=28456 dup_rows=134 dup_rows_remote=124 dedup_ratio=0.004089 router_tokens_per_expert=[2074, 2077, 2022, 2059, 2094, 2019, 2026, 2013] payload_rows_per_dst=[4178, 4240, 4226, 4236, 4002, 3818, 4032, 3902] dup_rows_per_dst=[10, 14, 46, 26, 14, 0, 24, 0] combine_rows_per_src=[4178, 4240, 4226, 4236, 4002, 3818, 4032, 3902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233111552 dispatch_meta_bytes=114320 combine_payload_bytes=233111552 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233225872 ingress_bytes=3051683840 link_bytes=3051683840 token_bytes=466337424 weight_movement_bytes=2818572288 total_nvlink_bytes=3284909712 weight_to_token_ratio=6.044062 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708815567e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28828 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2116, 2022, 2059, 1969, 1970, 2153, 2006, 2089] payload_rows_per_dst=[4232, 4044, 4118, 3938, 3940, 4306, 4012, 4178] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4232, 4044, 4118, 3938, 3940, 4306, 4012, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=28464 dup_rows=34 dup_rows_remote=24 dedup_ratio=0.001038 router_tokens_per_expert=[2031, 2037, 2034, 2073, 2038, 2035, 2040, 2096] payload_rows_per_dst=[3946, 3988, 3986, 4198, 4162, 4100, 4084, 4270] dup_rows_per_dst=[0, 0, 0, 8, 12, 4, 0, 10] combine_rows_per_src=[3946, 3988, 3986, 4198, 4162, 4100, 4084, 4270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2050, 2000, 2036, 2066, 2059, 2062, 2052] payload_rows_per_dst=[4118, 4100, 4000, 4072, 4132, 4118, 4124, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4100, 4000, 4072, 4132, 4118, 4124, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1972, 2045, 2043, 2096, 2125, 2062, 2017, 2024] payload_rows_per_dst=[3944, 4090, 4086, 4192, 4250, 4124, 4034, 4048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3944, 4090, 4086, 4192, 4250, 4124, 4034, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2050, 2078, 2013, 2003, 2106, 2035, 2086, 2013] payload_rows_per_dst=[4100, 4156, 4026, 4006, 4212, 4070, 4172, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4100, 4156, 4026, 4006, 4212, 4070, 4172, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28766 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2001, 2105, 2012, 2008, 2047, 2054, 2093] payload_rows_per_dst=[4128, 4002, 4210, 4024, 4016, 4094, 4108, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4002, 4210, 4024, 4016, 4094, 4108, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=28524 dup_rows=66 dup_rows_remote=64 dedup_ratio=0.002014 router_tokens_per_expert=[2066, 2070, 2050, 1993, 2005, 2085, 2001, 2114] payload_rows_per_dst=[4178, 4308, 4102, 3986, 4176, 4114, 3842, 3996] dup_rows_per_dst=[2, 22, 8, 0, 34, 0, 0, 0] combine_rows_per_src=[4178, 4308, 4102, 3986, 4176, 4114, 3842, 3996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28590 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2067, 2035, 2034, 2014, 2067, 2089, 2022] payload_rows_per_dst=[4112, 4134, 4070, 4068, 4028, 4134, 4178, 4044] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4134, 4070, 4068, 4028, 4134, 4178, 4044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233177088 dispatch_meta_bytes=113952 combine_payload_bytes=233177088 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233291040 ingress_bytes=2347106304 link_bytes=2347106304 token_bytes=466468128 weight_movement_bytes=2113929216 total_nvlink_bytes=2580397344 weight_to_token_ratio=4.531776 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314281557e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=236158976 dispatch_meta_bytes=115312 combine_payload_bytes=236158976 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236274288 ingress_bytes=2350088192 link_bytes=2350088192 token_bytes=472433264 weight_movement_bytes=2113929216 total_nvlink_bytes=2586362480 weight_to_token_ratio=4.474556 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315951290e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235667456 dispatch_meta_bytes=115072 combine_payload_bytes=235667456 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235782528 ingress_bytes=3054239744 link_bytes=3054239744 token_bytes=471449984 weight_movement_bytes=2818572288 total_nvlink_bytes=3290022272 weight_to_token_ratio=5.978518 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710246766e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234766224 ingress_bytes=3053223936 link_bytes=3053223936 token_bytes=469417872 weight_movement_bytes=2818572288 total_nvlink_bytes=3287990160 weight_to_token_ratio=6.004399 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709677956e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235618304 dispatch_meta_bytes=115048 combine_payload_bytes=235618304 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235733352 ingress_bytes=1644904448 link_bytes=1644904448 token_bytes=471351656 weight_movement_bytes=1409286144 total_nvlink_bytes=1880637800 weight_to_token_ratio=2.989883 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210778292e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235651072 dispatch_meta_bytes=115064 combine_payload_bytes=235651072 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235766136 ingress_bytes=3054223360 link_bytes=3054223360 token_bytes=471417208 weight_movement_bytes=2818572288 total_nvlink_bytes=3289989496 weight_to_token_ratio=5.978934 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710237592e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233668608 dispatch_meta_bytes=114352 combine_payload_bytes=233668608 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233782960 ingress_bytes=3052240896 link_bytes=3052240896 token_bytes=467451568 weight_movement_bytes=2818572288 total_nvlink_bytes=3286023856 weight_to_token_ratio=6.029656 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709127495e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234209280 dispatch_meta_bytes=114360 combine_payload_bytes=234209280 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234323640 ingress_bytes=3052781568 link_bytes=3052781568 token_bytes=468532920 weight_movement_bytes=2818572288 total_nvlink_bytes=3287105208 weight_to_token_ratio=6.015740 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709430249e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2043, 2021, 2058, 2036, 2036, 2067, 2052, 2071] payload_rows_per_dst=[4086, 4042, 4116, 4072, 4072, 4134, 4104, 4142] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4086, 4042, 4116, 4072, 4072, 4134, 4104, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32718 payload_rows_remote=28684 dup_rows=50 dup_rows_remote=48 dedup_ratio=0.001526 router_tokens_per_expert=[2007, 2033, 2067, 2129, 2031, 2069, 2038, 2010] payload_rows_per_dst=[3844, 4020, 3980, 4382, 4168, 4214, 4076, 4034] dup_rows_per_dst=[0, 0, 0, 18, 18, 12, 0, 2] combine_rows_per_src=[3844, 4020, 3980, 4382, 4168, 4214, 4076, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32640 payload_rows_remote=28248 dup_rows=128 dup_rows_remote=84 dedup_ratio=0.003906 router_tokens_per_expert=[2080, 2070, 2094, 2051, 2048, 2013, 2012, 2016] payload_rows_per_dst=[4392, 4528, 4054, 4172, 3644, 3884, 4084, 3882] dup_rows_per_dst=[44, 64, 0, 16, 0, 0, 4, 0] combine_rows_per_src=[4392, 4528, 4054, 4172, 3644, 3884, 4084, 3882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2065, 2032, 2058, 2002, 2067, 2078, 2017] payload_rows_per_dst=[4130, 4130, 4064, 4116, 4004, 4134, 4156, 4034] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4130, 4064, 4116, 4004, 4134, 4156, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2106, 1953, 2061, 1995, 2074, 2053, 2118, 2024] payload_rows_per_dst=[4212, 3906, 4122, 3990, 4148, 4106, 4236, 4048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4212, 3906, 4122, 3990, 4148, 4106, 4236, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28616 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1992, 2076, 2016, 2069, 2012, 2075, 2086, 2058] payload_rows_per_dst=[3984, 4152, 4032, 4138, 4024, 4150, 4172, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3984, 4152, 4032, 4138, 4024, 4150, 4172, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2089, 2033, 2053, 1978, 2032, 1972, 2082, 2145] payload_rows_per_dst=[4178, 4066, 4106, 3956, 4064, 3944, 4164, 4290] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4178, 4066, 4106, 3956, 4064, 3944, 4164, 4290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235192416 ingress_bytes=1644363776 link_bytes=1644363776 token_bytes=470270048 weight_movement_bytes=1409286144 total_nvlink_bytes=1879556192 weight_to_token_ratio=2.996759 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207750754e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114928 combine_payload_bytes=234979328 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235094256 ingress_bytes=3053551616 link_bytes=3053551616 token_bytes=470073584 weight_movement_bytes=2818572288 total_nvlink_bytes=3288645872 weight_to_token_ratio=5.996024 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709861443e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=231407616 dispatch_meta_bytes=113328 combine_payload_bytes=231407616 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231520944 ingress_bytes=1640693760 link_bytes=1640693760 token_bytes=462928560 weight_movement_bytes=1409286144 total_nvlink_bytes=1872214704 weight_to_token_ratio=3.044284 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.187200197e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2077, 2061, 2119, 2055, 2030, 1980, 2009, 2053] payload_rows_per_dst=[4154, 4122, 4238, 4110, 4060, 3960, 4018, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4154, 4122, 4238, 4110, 4060, 3960, 4018, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234831792 ingress_bytes=2348646400 link_bytes=2348646400 token_bytes=469548976 weight_movement_bytes=2113929216 total_nvlink_bytes=2583478192 weight_to_token_ratio=4.502042 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315143947e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234913752 ingress_bytes=3758014464 link_bytes=3758014464 token_bytes=469712856 weight_movement_bytes=3523215360 total_nvlink_bytes=3992928216 weight_to_token_ratio=7.500785 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104331233e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234422272 dispatch_meta_bytes=114464 combine_payload_bytes=234422272 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234536736 ingress_bytes=2348351488 link_bytes=2348351488 token_bytes=468959008 weight_movement_bytes=2113929216 total_nvlink_bytes=2582888224 weight_to_token_ratio=4.507706 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314978808e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234913752 ingress_bytes=3053371392 link_bytes=3053371392 token_bytes=469712856 weight_movement_bytes=2818572288 total_nvlink_bytes=3288285144 weight_to_token_ratio=6.000628 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709760526e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235635000 ingress_bytes=2349449216 link_bytes=2349449216 token_bytes=471155000 weight_movement_bytes=2113929216 total_nvlink_bytes=2585084216 weight_to_token_ratio=4.486696 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315593490e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28626 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 2044, 2058, 2078, 2071, 2068, 2014, 2016] payload_rows_per_dst=[4070, 4088, 4116, 4156, 4142, 4136, 4028, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 4088, 4116, 4156, 4142, 4136, 4028, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32724 payload_rows_remote=28568 dup_rows=44 dup_rows_remote=14 dedup_ratio=0.001343 router_tokens_per_expert=[1973, 2040, 2057, 2068, 2049, 2111, 2056, 2030] payload_rows_per_dst=[4156, 3940, 4120, 4136, 4176, 4126, 4010, 4060] dup_rows_per_dst=[30, 0, 0, 0, 14, 0, 0, 0] combine_rows_per_src=[4156, 3940, 4120, 4136, 4176, 4126, 4010, 4060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32746 payload_rows_remote=28500 dup_rows=22 dup_rows_remote=20 dedup_ratio=0.000671 router_tokens_per_expert=[2077, 2052, 2036, 1996, 2047, 1997, 2062, 2117] payload_rows_per_dst=[4154, 4104, 3990, 3992, 4108, 4018, 4134, 4246] dup_rows_per_dst=[0, 0, 0, 0, 10, 4, 6, 2] combine_rows_per_src=[4154, 4104, 3990, 3992, 4108, 4018, 4134, 4246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28706 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2005, 2019, 2069, 2031, 2150, 2035, 2069, 2006] payload_rows_per_dst=[4010, 4038, 4138, 4062, 4300, 4070, 4138, 4012] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4010, 4038, 4138, 4062, 4300, 4070, 4138, 4012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 1995, 2060, 1983, 2082, 2067, 2106, 2040] payload_rows_per_dst=[4102, 3990, 4120, 3966, 4164, 4134, 4212, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 3990, 4120, 3966, 4164, 4134, 4212, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28556 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 1975, 2067, 2007, 2129, 2106, 1997, 2079] payload_rows_per_dst=[4048, 3950, 4134, 4014, 4258, 4212, 3994, 4158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 3950, 4134, 4014, 4258, 4212, 3994, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2023, 2032, 2031, 2080, 2055, 2054, 2016, 2093] payload_rows_per_dst=[4046, 4064, 4062, 4160, 4110, 4108, 4032, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4046, 4064, 4062, 4160, 4110, 4108, 4032, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234504192 dispatch_meta_bytes=114504 combine_payload_bytes=234504192 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234618696 ingress_bytes=3757719552 link_bytes=3757719552 token_bytes=469122888 weight_movement_bytes=3523215360 total_nvlink_bytes=3992338248 weight_to_token_ratio=7.510218 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104166094e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234029056 dispatch_meta_bytes=114328 combine_payload_bytes=234029056 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234143384 ingress_bytes=3052601344 link_bytes=3052601344 token_bytes=468172440 weight_movement_bytes=2818572288 total_nvlink_bytes=3286744728 weight_to_token_ratio=6.020372 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709329331e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114080 combine_payload_bytes=233472000 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233586080 ingress_bytes=3756687360 link_bytes=3756687360 token_bytes=467058080 weight_movement_bytes=3523215360 total_nvlink_bytes=3990273440 weight_to_token_ratio=7.543420 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103588110e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235159552 dispatch_meta_bytes=114824 combine_payload_bytes=235159552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235274376 ingress_bytes=1644445696 link_bytes=1644445696 token_bytes=470433928 weight_movement_bytes=1409286144 total_nvlink_bytes=1879720072 weight_to_token_ratio=2.995715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208209472e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=234799008 ingress_bytes=3757899776 link_bytes=3757899776 token_bytes=469483424 weight_movement_bytes=3523215360 total_nvlink_bytes=3992698784 weight_to_token_ratio=7.504451 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104267012e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114224 combine_payload_bytes=233930752 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234044976 ingress_bytes=2347859968 link_bytes=2347859968 token_bytes=467975728 weight_movement_bytes=2113929216 total_nvlink_bytes=2581904944 weight_to_token_ratio=4.517177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314703578e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235257984 ingress_bytes=1644429312 link_bytes=1644429312 token_bytes=470401152 weight_movement_bytes=1409286144 total_nvlink_bytes=1879687296 weight_to_token_ratio=2.995924 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208117729e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1951, 1971, 2077, 2115, 2016, 2101, 2068, 2085] payload_rows_per_dst=[3902, 3942, 4154, 4230, 4032, 4202, 4136, 4170] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3902, 3942, 4154, 4230, 4032, 4202, 4136, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28250 dup_rows=80 dup_rows_remote=20 dedup_ratio=0.002441 router_tokens_per_expert=[2083, 2065, 2046, 2033, 2074, 1992, 2042, 2049] payload_rows_per_dst=[4438, 4252, 4064, 4066, 4006, 3984, 4084, 3794] dup_rows_per_dst=[60, 20, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4438, 4252, 4064, 4066, 4006, 3984, 4084, 3794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=28368 dup_rows=76 dup_rows_remote=20 dedup_ratio=0.002319 router_tokens_per_expert=[2055, 2078, 2073, 2005, 2055, 2066, 2041, 2011] payload_rows_per_dst=[4042, 4156, 4044, 4016, 4160, 4106, 3844, 4324] dup_rows_per_dst=[0, 0, 0, 10, 6, 4, 0, 56] combine_rows_per_src=[4042, 4156, 4044, 4016, 4160, 4106, 3844, 4324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28608 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2151, 1983, 2045, 2080, 2091, 2030, 1958] payload_rows_per_dst=[4092, 4302, 3966, 4090, 4160, 4182, 4060, 3916] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4302, 3966, 4090, 4160, 4182, 4060, 3916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2062, 2008, 2044, 2053, 2012, 2077, 2086] payload_rows_per_dst=[4084, 4124, 4016, 4088, 4106, 4024, 4154, 4172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4124, 4016, 4088, 4106, 4024, 4154, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28518 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2010, 2054, 2125, 2109, 1982, 1978, 2064, 2062] payload_rows_per_dst=[4020, 4108, 4250, 4218, 3964, 3956, 4128, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4020, 4108, 4250, 4218, 3964, 3956, 4128, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28776 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2025, 2014, 2115, 2067, 2037, 1996, 2084, 2046] payload_rows_per_dst=[4050, 4028, 4230, 4134, 4074, 3992, 4168, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4050, 4028, 4230, 4134, 4074, 3992, 4168, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28874 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2100, 1947, 2018, 2028, 2039, 2151, 2117, 1984] payload_rows_per_dst=[4200, 3894, 4036, 4056, 4078, 4302, 4234, 3968] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4200, 3894, 4036, 4056, 4078, 4302, 4234, 3968] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234667872 ingress_bytes=1643839488 link_bytes=1643839488 token_bytes=469221216 weight_movement_bytes=1409286144 total_nvlink_bytes=1878507360 weight_to_token_ratio=3.003458 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204814960e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=232390656 dispatch_meta_bytes=113552 combine_payload_bytes=232390656 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232504208 ingress_bytes=1641676800 link_bytes=1641676800 token_bytes=464894864 weight_movement_bytes=1409286144 total_nvlink_bytes=1874181008 weight_to_token_ratio=3.031408 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.192704810e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=231424000 dispatch_meta_bytes=113080 combine_payload_bytes=231424000 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231537080 ingress_bytes=2345353216 link_bytes=2345353216 token_bytes=462961080 weight_movement_bytes=2113929216 total_nvlink_bytes=2576890296 weight_to_token_ratio=4.566106 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313299901e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235061280 ingress_bytes=2348875776 link_bytes=2348875776 token_bytes=470007840 weight_movement_bytes=2113929216 total_nvlink_bytes=2583937056 weight_to_token_ratio=4.497647 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315272388e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234356736 dispatch_meta_bytes=114432 combine_payload_bytes=234356736 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234471168 ingress_bytes=1643642880 link_bytes=1643642880 token_bytes=468827904 weight_movement_bytes=1409286144 total_nvlink_bytes=1878114048 weight_to_token_ratio=3.005978 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203714038e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233619456 dispatch_meta_bytes=114072 combine_payload_bytes=233619456 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=233733528 ingress_bytes=4461477888 link_bytes=4461477888 token_bytes=467352984 weight_movement_bytes=4227858432 total_nvlink_bytes=4695211416 weight_to_token_ratio=9.046392 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498241386e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235732992 dispatch_meta_bytes=115104 combine_payload_bytes=235732992 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235848096 ingress_bytes=3054305280 link_bytes=3054305280 token_bytes=471581088 weight_movement_bytes=2818572288 total_nvlink_bytes=3290153376 weight_to_token_ratio=5.976856 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710283464e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=236535808 dispatch_meta_bytes=115496 combine_payload_bytes=236535808 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236651304 ingress_bytes=2350465024 link_bytes=2350465024 token_bytes=473187112 weight_movement_bytes=2113929216 total_nvlink_bytes=2587116328 weight_to_token_ratio=4.467428 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.316162300e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28598 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 2108, 2053, 2010, 2015, 2038, 2085, 2028] payload_rows_per_dst=[4094, 4216, 4106, 4020, 4030, 4076, 4170, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 4216, 4106, 4020, 4030, 4076, 4170, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28350 dup_rows=84 dup_rows_remote=62 dedup_ratio=0.002563 router_tokens_per_expert=[2055, 2034, 1952, 2039, 2155, 1991, 2064, 2094] payload_rows_per_dst=[4054, 3970, 3704, 3684, 4310, 4266, 4362, 4334] dup_rows_per_dst=[0, 0, 0, 0, 0, 22, 40, 22] combine_rows_per_src=[4054, 3970, 3704, 3684, 4310, 4266, 4362, 4334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=28568 dup_rows=96 dup_rows_remote=84 dedup_ratio=0.002930 router_tokens_per_expert=[2014, 2029, 2032, 2082, 2057, 2047, 2070, 2053] payload_rows_per_dst=[4104, 4000, 4032, 4438, 4136, 4078, 4056, 3828] dup_rows_per_dst=[12, 2, 0, 44, 20, 0, 18, 0] combine_rows_per_src=[4104, 4000, 4032, 4438, 4136, 4078, 4056, 3828] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28610 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2050, 2087, 2079, 2016, 2048, 2042, 2028] payload_rows_per_dst=[4068, 4100, 4174, 4158, 4032, 4096, 4084, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4100, 4174, 4158, 4032, 4096, 4084, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1968, 2028, 2102, 2070, 2111, 1978, 1987, 2140] payload_rows_per_dst=[3936, 4056, 4204, 4140, 4222, 3956, 3974, 4280] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3936, 4056, 4204, 4140, 4222, 3956, 3974, 4280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28720 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2079, 2024, 2079, 2099, 2010, 2064, 2013, 2016] payload_rows_per_dst=[4158, 4048, 4158, 4198, 4020, 4128, 4026, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4158, 4048, 4158, 4198, 4020, 4128, 4026, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1985, 2147, 2051, 2053, 2011, 2048, 2047, 2042] payload_rows_per_dst=[3970, 4294, 4102, 4106, 4022, 4096, 4094, 4084] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3970, 4294, 4102, 4106, 4022, 4096, 4094, 4084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28724 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2013, 1959, 2104, 2108, 2022, 2083, 2057, 2038] payload_rows_per_dst=[4026, 3918, 4208, 4216, 4044, 4166, 4114, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4026, 3918, 4208, 4216, 4044, 4166, 4114, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114392 combine_payload_bytes=234274816 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234389208 ingress_bytes=3052847104 link_bytes=3052847104 token_bytes=468664024 weight_movement_bytes=2818572288 total_nvlink_bytes=3287236312 weight_to_token_ratio=6.014057 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709466946e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234373120 dispatch_meta_bytes=114440 combine_payload_bytes=234373120 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234487560 ingress_bytes=4462231552 link_bytes=4462231552 token_bytes=468860680 weight_movement_bytes=4227858432 total_nvlink_bytes=4696719112 weight_to_token_ratio=9.017302 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498663406e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=232243200 dispatch_meta_bytes=113648 combine_payload_bytes=232243200 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232356848 ingress_bytes=2346172416 link_bytes=2346172416 token_bytes=464600048 weight_movement_bytes=2113929216 total_nvlink_bytes=2578529264 weight_to_token_ratio=4.549998 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313758619e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234029056 dispatch_meta_bytes=114608 combine_payload_bytes=234029056 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234143664 ingress_bytes=2347958272 link_bytes=2347958272 token_bytes=468172720 weight_movement_bytes=2113929216 total_nvlink_bytes=2582101936 weight_to_token_ratio=4.515276 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314758624e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233996288 dispatch_meta_bytes=114256 combine_payload_bytes=233996288 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234110544 ingress_bytes=3052568576 link_bytes=3052568576 token_bytes=468106832 weight_movement_bytes=2818572288 total_nvlink_bytes=3286679120 weight_to_token_ratio=6.021216 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709310982e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235274240 dispatch_meta_bytes=114880 combine_payload_bytes=235274240 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235389120 ingress_bytes=3053846528 link_bytes=3053846528 token_bytes=470663360 weight_movement_bytes=2818572288 total_nvlink_bytes=3289235648 weight_to_token_ratio=5.988510 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710026582e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234995712 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=469876736 weight_movement_bytes=1409286144 total_nvlink_bytes=1879162880 weight_to_token_ratio=2.999268 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235307008 dispatch_meta_bytes=114896 combine_payload_bytes=235307008 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=235421904 ingress_bytes=1644593152 link_bytes=1644593152 token_bytes=470728912 weight_movement_bytes=1409286144 total_nvlink_bytes=1880015056 weight_to_token_ratio=2.993838 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209035164e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28778 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2055, 2077, 2100, 2056, 2027, 1995, 2045] payload_rows_per_dst=[4058, 4110, 4154, 4200, 4112, 4054, 3990, 4090] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4110, 4154, 4200, 4112, 4054, 3990, 4090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28790 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2020, 2092, 1986, 1989, 2087, 2025, 2128, 2057] payload_rows_per_dst=[4040, 4184, 3972, 3978, 4174, 4050, 4256, 4114] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4040, 4184, 3972, 3978, 4174, 4050, 4256, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32718 payload_rows_remote=28580 dup_rows=50 dup_rows_remote=42 dedup_ratio=0.001526 router_tokens_per_expert=[2035, 2060, 2090, 2047, 2122, 1995, 1981, 2054] payload_rows_per_dst=[3784, 4098, 4180, 4094, 4244, 4218, 3962, 4138] dup_rows_per_dst=[0, 0, 0, 0, 0, 42, 0, 8] combine_rows_per_src=[3784, 4098, 4180, 4094, 4244, 4218, 3962, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=28468 dup_rows=76 dup_rows_remote=44 dedup_ratio=0.002319 router_tokens_per_expert=[2075, 2038, 2003, 2029, 2031, 2118, 2041, 2049] payload_rows_per_dst=[4224, 4236, 3730, 4108, 4062, 4232, 4160, 3940] dup_rows_per_dst=[32, 14, 0, 16, 0, 0, 14, 0] combine_rows_per_src=[4224, 4236, 3730, 4108, 4062, 4232, 4160, 3940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28746 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2107, 2033, 2011, 2013, 2041, 2045, 2051, 2083] payload_rows_per_dst=[4214, 4066, 4022, 4026, 4082, 4090, 4102, 4166] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4214, 4066, 4022, 4026, 4082, 4090, 4102, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28708 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1994, 2030, 2065, 2042, 2076, 2059, 2024, 2094] payload_rows_per_dst=[3988, 4060, 4130, 4084, 4152, 4118, 4048, 4188] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3988, 4060, 4130, 4084, 4152, 4118, 4048, 4188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235749376 dispatch_meta_bytes=115112 combine_payload_bytes=235749376 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235864488 ingress_bytes=1645035520 link_bytes=1645035520 token_bytes=471613864 weight_movement_bytes=1409286144 total_nvlink_bytes=1880900008 weight_to_token_ratio=2.988220 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211512240e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235847680 dispatch_meta_bytes=115160 combine_payload_bytes=235847680 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235962840 ingress_bytes=1645133824 link_bytes=1645133824 token_bytes=471810520 weight_movement_bytes=1409286144 total_nvlink_bytes=1881096664 weight_to_token_ratio=2.986975 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212062702e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28758 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2070, 2118, 2094, 2002, 1980, 2005, 2062, 2053] payload_rows_per_dst=[4140, 4236, 4188, 4004, 3960, 4010, 4124, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4236, 4188, 4004, 3960, 4010, 4124, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28808 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2096, 2047, 2064, 2032, 1980, 2029, 2084, 2052] payload_rows_per_dst=[4192, 4094, 4128, 4064, 3960, 4058, 4168, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4192, 4094, 4128, 4064, 3960, 4058, 4168, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234127360 dispatch_meta_bytes=114488 combine_payload_bytes=234127360 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234241848 ingress_bytes=3757342720 link_bytes=3757342720 token_bytes=468369208 weight_movement_bytes=3523215360 total_nvlink_bytes=3991584568 weight_to_token_ratio=7.522304 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103955084e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233209856 dispatch_meta_bytes=114048 combine_payload_bytes=233209856 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233323904 ingress_bytes=3051782144 link_bytes=3051782144 token_bytes=466533760 weight_movement_bytes=2818572288 total_nvlink_bytes=3285106048 weight_to_token_ratio=6.041518 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708870613e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235487232 dispatch_meta_bytes=114984 combine_payload_bytes=235487232 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=235602216 ingress_bytes=1644773376 link_bytes=1644773376 token_bytes=471089448 weight_movement_bytes=1409286144 total_nvlink_bytes=1880375592 weight_to_token_ratio=2.991547 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210044343e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235175936 dispatch_meta_bytes=114832 combine_payload_bytes=235175936 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235290768 ingress_bytes=2349105152 link_bytes=2349105152 token_bytes=470466704 weight_movement_bytes=2113929216 total_nvlink_bytes=2584395920 weight_to_token_ratio=4.493260 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315400829e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235585536 dispatch_meta_bytes=115032 combine_payload_bytes=235585536 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235700568 ingress_bytes=2349514752 link_bytes=2349514752 token_bytes=471286104 weight_movement_bytes=2113929216 total_nvlink_bytes=2585215320 weight_to_token_ratio=4.485448 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315630188e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235995136 dispatch_meta_bytes=115232 combine_payload_bytes=235995136 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236110368 ingress_bytes=2349924352 link_bytes=2349924352 token_bytes=472105504 weight_movement_bytes=2113929216 total_nvlink_bytes=2586034720 weight_to_token_ratio=4.477663 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315859546e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2087, 1981, 2113, 2073, 2047, 2086, 1958, 2039] payload_rows_per_dst=[4174, 3962, 4226, 4146, 4094, 4172, 3916, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4174, 3962, 4226, 4146, 4094, 4172, 3916, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2147, 2014, 2058, 2018, 2020, 2052, 2017, 2058] payload_rows_per_dst=[4294, 4028, 4116, 4036, 4040, 4104, 4034, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4294, 4028, 4116, 4036, 4040, 4104, 4034, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28462 dup_rows=80 dup_rows_remote=72 dedup_ratio=0.002441 router_tokens_per_expert=[1989, 2086, 2043, 2066, 2089, 2053, 1970, 2088] payload_rows_per_dst=[3816, 3948, 4082, 4078, 4348, 4140, 4050, 4226] dup_rows_per_dst=[0, 0, 0, 4, 30, 28, 10, 8] combine_rows_per_src=[3816, 3948, 4082, 4078, 4348, 4140, 4050, 4226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=28536 dup_rows=52 dup_rows_remote=2 dedup_ratio=0.001587 router_tokens_per_expert=[1970, 2024, 2075, 1986, 2080, 2155, 2077, 2017] payload_rows_per_dst=[4180, 4048, 4154, 3972, 4160, 4208, 3960, 4034] dup_rows_per_dst=[50, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[4180, 4048, 4154, 3972, 4160, 4208, 3960, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114488 combine_payload_bytes=234471424 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234585912 ingress_bytes=2348400640 link_bytes=2348400640 token_bytes=469057336 weight_movement_bytes=2113929216 total_nvlink_bytes=2582986552 weight_to_token_ratio=4.506761 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315006331e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2069, 2076, 2063, 2029, 2013, 2060, 2071, 2003] payload_rows_per_dst=[4138, 4152, 4126, 4058, 4026, 4120, 4142, 4006] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4138, 4152, 4126, 4058, 4026, 4120, 4142, 4006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2010, 2058, 2043, 2072, 1988, 2063, 2115, 2035] payload_rows_per_dst=[4020, 4116, 4086, 4144, 3976, 4126, 4230, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4020, 4116, 4086, 4144, 3976, 4126, 4230, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235503864 ingress_bytes=2349318144 link_bytes=2349318144 token_bytes=470892792 weight_movement_bytes=2113929216 total_nvlink_bytes=2584822008 weight_to_token_ratio=4.489194 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315520095e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28810 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2102, 2118, 2021, 2049, 1979, 2049, 2028] payload_rows_per_dst=[4076, 4204, 4236, 4042, 4098, 3958, 4098, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4204, 4236, 4042, 4098, 3958, 4098, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28778 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2053, 2097, 2040, 1995, 1971, 2076, 2122] payload_rows_per_dst=[4060, 4106, 4194, 4080, 3990, 3942, 4152, 4244] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4106, 4194, 4080, 3990, 3942, 4152, 4244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233160704 dispatch_meta_bytes=114136 combine_payload_bytes=233160704 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233274840 ingress_bytes=3051732992 link_bytes=3051732992 token_bytes=466435544 weight_movement_bytes=2818572288 total_nvlink_bytes=3285007832 weight_to_token_ratio=6.042791 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708843090e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233766912 dispatch_meta_bytes=114152 combine_payload_bytes=233766912 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233881064 ingress_bytes=1643053056 link_bytes=1643053056 token_bytes=467647976 weight_movement_bytes=1409286144 total_nvlink_bytes=1876934120 weight_to_token_ratio=3.013562 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.200411270e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114568 combine_payload_bytes=234635264 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234749832 ingress_bytes=3757850624 link_bytes=3757850624 token_bytes=469385096 weight_movement_bytes=3523215360 total_nvlink_bytes=3992600456 weight_to_token_ratio=7.506023 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104239489e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234831792 ingress_bytes=2348646400 link_bytes=2348646400 token_bytes=469548976 weight_movement_bytes=2113929216 total_nvlink_bytes=2583478192 weight_to_token_ratio=4.502042 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315143947e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=236011520 dispatch_meta_bytes=115240 combine_payload_bytes=236011520 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=236126760 ingress_bytes=3054583808 link_bytes=3054583808 token_bytes=472138280 weight_movement_bytes=2818572288 total_nvlink_bytes=3290710568 weight_to_token_ratio=5.969803 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710439428e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2047, 2015, 2040, 2087, 2095, 2030, 2013] payload_rows_per_dst=[4114, 4094, 4030, 4080, 4174, 4190, 4060, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4094, 4030, 4080, 4174, 4190, 4060, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235749376 dispatch_meta_bytes=115112 combine_payload_bytes=235749376 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235864488 ingress_bytes=2349678592 link_bytes=2349678592 token_bytes=471613864 weight_movement_bytes=2113929216 total_nvlink_bytes=2585543080 weight_to_token_ratio=4.482331 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315721931e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28772 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2114, 2015, 2044, 2046, 2072, 2025, 1998, 2070] payload_rows_per_dst=[4228, 4030, 4088, 4092, 4144, 4050, 3996, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4228, 4030, 4088, 4092, 4144, 4050, 3996, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235126848 ingress_bytes=1644298240 link_bytes=1644298240 token_bytes=470138944 weight_movement_bytes=1409286144 total_nvlink_bytes=1879425088 weight_to_token_ratio=2.997595 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207383780e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=28246 dup_rows=98 dup_rows_remote=30 dedup_ratio=0.002991 router_tokens_per_expert=[2012, 2090, 2070, 2075, 2056, 2028, 2039, 2014] payload_rows_per_dst=[4024, 3928, 4052, 4296, 4158, 3754, 4034, 4424] dup_rows_per_dst=[0, 0, 2, 22, 6, 0, 0, 68] combine_rows_per_src=[4024, 3928, 4052, 4296, 4158, 3754, 4034, 4424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32678 payload_rows_remote=28276 dup_rows=90 dup_rows_remote=22 dedup_ratio=0.002747 router_tokens_per_expert=[2042, 2003, 2042, 2031, 2125, 1958, 2098, 2085] payload_rows_per_dst=[4402, 4226, 3924, 3728, 4244, 3916, 4124, 4114] dup_rows_per_dst=[68, 22, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4402, 4226, 3924, 3728, 4244, 3916, 4124, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2095, 2057, 2095, 2045, 1986, 2037, 2100, 1969] payload_rows_per_dst=[4190, 4114, 4190, 4090, 3972, 4074, 4200, 3938] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4190, 4114, 4190, 4090, 3972, 4074, 4200, 3938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235700224 dispatch_meta_bytes=115088 combine_payload_bytes=235700224 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235815312 ingress_bytes=1644986368 link_bytes=1644986368 token_bytes=471515536 weight_movement_bytes=1409286144 total_nvlink_bytes=1880801680 weight_to_token_ratio=2.988843 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211237010e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28710 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2012, 2029, 2036, 2070, 2043, 2110, 2045, 2039] payload_rows_per_dst=[4024, 4058, 4072, 4140, 4086, 4220, 4090, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 4058, 4072, 4140, 4086, 4220, 4090, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1976, 2107, 2045, 2032, 2047, 2097, 2032, 2048] payload_rows_per_dst=[3952, 4214, 4090, 4064, 4094, 4194, 4064, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3952, 4214, 4090, 4064, 4094, 4194, 4064, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 2107, 2032, 1991, 2038, 2043, 2112, 2026] payload_rows_per_dst=[4070, 4214, 4064, 3982, 4076, 4086, 4224, 4052] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 4214, 4064, 3982, 4076, 4086, 4224, 4052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=231391232 dispatch_meta_bytes=113104 combine_payload_bytes=231391232 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231504336 ingress_bytes=1640677376 link_bytes=1640677376 token_bytes=462895568 weight_movement_bytes=1409286144 total_nvlink_bytes=1872181712 weight_to_token_ratio=3.044501 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.187108453e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=231636992 dispatch_meta_bytes=113192 combine_payload_bytes=231636992 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231750184 ingress_bytes=1640923136 link_bytes=1640923136 token_bytes=463387176 weight_movement_bytes=1409286144 total_nvlink_bytes=1872673320 weight_to_token_ratio=3.041271 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.188484607e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114312 combine_payload_bytes=234110976 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234225288 ingress_bytes=3052683264 link_bytes=3052683264 token_bytes=468336264 weight_movement_bytes=2818572288 total_nvlink_bytes=3286908552 weight_to_token_ratio=6.018266 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709375203e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235192320 dispatch_meta_bytes=114840 combine_payload_bytes=235192320 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=235307160 ingress_bytes=3758407680 link_bytes=3758407680 token_bytes=470499480 weight_movement_bytes=3523215360 total_nvlink_bytes=3993714840 weight_to_token_ratio=7.488245 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104551417e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234078208 dispatch_meta_bytes=114296 combine_payload_bytes=234078208 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234192504 ingress_bytes=3052650496 link_bytes=3052650496 token_bytes=468270712 weight_movement_bytes=2818572288 total_nvlink_bytes=3286843000 weight_to_token_ratio=6.019109 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709356854e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28668 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1974, 2059, 2131, 2050, 2055, 2016, 2052, 2047] payload_rows_per_dst=[3948, 4118, 4262, 4100, 4110, 4032, 4104, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3948, 4118, 4262, 4100, 4110, 4032, 4104, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235159632 ingress_bytes=1644331008 link_bytes=1644331008 token_bytes=470204496 weight_movement_bytes=1409286144 total_nvlink_bytes=1879490640 weight_to_token_ratio=2.997177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207567267e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2082, 2049, 2056, 2061, 2021, 2021, 2068, 2026] payload_rows_per_dst=[4164, 4098, 4112, 4122, 4042, 4042, 4136, 4052] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4164, 4098, 4112, 4122, 4042, 4042, 4136, 4052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234848256 dispatch_meta_bytes=114672 combine_payload_bytes=234848256 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234962928 ingress_bytes=3758063616 link_bytes=3758063616 token_bytes=469811184 weight_movement_bytes=3523215360 total_nvlink_bytes=3993026544 weight_to_token_ratio=7.499216 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104358756e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28394 dup_rows=62 dup_rows_remote=46 dedup_ratio=0.001892 router_tokens_per_expert=[1976, 2070, 2088, 2139, 1966, 2047, 2006, 2092] payload_rows_per_dst=[3952, 4140, 3944, 4086, 3994, 4238, 4040, 4312] dup_rows_per_dst=[0, 0, 0, 0, 12, 26, 8, 16] combine_rows_per_src=[3952, 4140, 3944, 4086, 3994, 4238, 4040, 4312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234667872 ingress_bytes=2348482560 link_bytes=2348482560 token_bytes=469221216 weight_movement_bytes=2113929216 total_nvlink_bytes=2583150432 weight_to_token_ratio=4.505187 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315052203e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=28384 dup_rows=58 dup_rows_remote=4 dedup_ratio=0.001770 router_tokens_per_expert=[2028, 2040, 2093, 2064, 2031, 2008, 2027, 2093] payload_rows_per_dst=[4326, 4058, 4186, 4132, 4062, 4000, 3802, 4144] dup_rows_per_dst=[54, 0, 0, 0, 0, 4, 0, 0] combine_rows_per_src=[4326, 4058, 4186, 4132, 4062, 4000, 3802, 4144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28610 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2089, 2079, 2068, 2022, 2064, 2005, 1999] payload_rows_per_dst=[4116, 4178, 4158, 4136, 4044, 4128, 4010, 3998] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4178, 4158, 4136, 4044, 4128, 4010, 3998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28828 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2040, 1970, 2068, 2138, 2029, 2075, 2060, 2004] payload_rows_per_dst=[4080, 3940, 4136, 4276, 4058, 4150, 4120, 4008] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4080, 3940, 4136, 4276, 4058, 4150, 4120, 4008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 2018, 1990, 2097, 2030, 2028, 2077, 2084] payload_rows_per_dst=[4120, 4036, 3980, 4194, 4060, 4056, 4154, 4168] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 4036, 3980, 4194, 4060, 4056, 4154, 4168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=232603648 dispatch_meta_bytes=113760 combine_payload_bytes=232603648 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=232717408 ingress_bytes=1641889792 link_bytes=1641889792 token_bytes=465321056 weight_movement_bytes=1409286144 total_nvlink_bytes=1874607200 weight_to_token_ratio=3.028632 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.193897477e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234373120 dispatch_meta_bytes=114440 combine_payload_bytes=234373120 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234487560 ingress_bytes=3052945408 link_bytes=3052945408 token_bytes=468860680 weight_movement_bytes=2818572288 total_nvlink_bytes=3287432968 weight_to_token_ratio=6.011535 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709521992e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=232521728 dispatch_meta_bytes=113552 combine_payload_bytes=232521728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232635280 ingress_bytes=1641807872 link_bytes=1641807872 token_bytes=465157008 weight_movement_bytes=1409286144 total_nvlink_bytes=1874443152 weight_to_token_ratio=3.029700 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.193438759e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28626 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2041, 2081, 2063, 2050, 2062, 2071, 1995] payload_rows_per_dst=[4042, 4082, 4162, 4126, 4100, 4124, 4142, 3990] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4082, 4162, 4126, 4100, 4124, 4142, 3990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2020, 2013, 2101, 2061, 2073, 2049, 2033, 2034] payload_rows_per_dst=[4040, 4026, 4202, 4122, 4146, 4098, 4066, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4040, 4026, 4202, 4122, 4146, 4098, 4066, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28754 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2054, 2023, 2007, 2016, 2049, 2073, 2146] payload_rows_per_dst=[4032, 4108, 4046, 4014, 4032, 4098, 4146, 4292] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4108, 4046, 4014, 4032, 4098, 4146, 4292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=236158976 dispatch_meta_bytes=115312 combine_payload_bytes=236158976 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236274288 ingress_bytes=1645445120 link_bytes=1645445120 token_bytes=472433264 weight_movement_bytes=1409286144 total_nvlink_bytes=1881719408 weight_to_token_ratio=2.983038 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213805829e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28498 dup_rows=84 dup_rows_remote=80 dedup_ratio=0.002563 router_tokens_per_expert=[2016, 2110, 2012, 2072, 2045, 2055, 1990, 2084] payload_rows_per_dst=[4032, 3914, 3868, 4270, 4348, 3972, 4094, 4186] dup_rows_per_dst=[0, 0, 14, 10, 40, 0, 16, 4] combine_rows_per_src=[4032, 3914, 3868, 4270, 4348, 3972, 4094, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235323552 ingress_bytes=2349137920 link_bytes=2349137920 token_bytes=470532256 weight_movement_bytes=2113929216 total_nvlink_bytes=2584461472 weight_to_token_ratio=4.492634 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315419177e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28708 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2098, 2088, 2030, 2062, 2049, 2041, 1984, 2032] payload_rows_per_dst=[4196, 4176, 4060, 4124, 4098, 4082, 3968, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4196, 4176, 4060, 4124, 4098, 4082, 3968, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234504192 dispatch_meta_bytes=114504 combine_payload_bytes=234504192 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234618696 ingress_bytes=2348433408 link_bytes=2348433408 token_bytes=469122888 weight_movement_bytes=2113929216 total_nvlink_bytes=2583052104 weight_to_token_ratio=4.506131 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315024680e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32726 payload_rows_remote=28512 dup_rows=42 dup_rows_remote=40 dedup_ratio=0.001282 router_tokens_per_expert=[2098, 2066, 2014, 2084, 2079, 2029, 1990, 2024] payload_rows_per_dst=[4214, 4132, 4110, 4046, 4256, 4058, 3862, 4048] dup_rows_per_dst=[2, 0, 20, 0, 20, 0, 0, 0] combine_rows_per_src=[4214, 4132, 4110, 4046, 4256, 4058, 3862, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114488 combine_payload_bytes=234471424 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234585912 ingress_bytes=2348400640 link_bytes=2348400640 token_bytes=469057336 weight_movement_bytes=2113929216 total_nvlink_bytes=2582986552 weight_to_token_ratio=4.506761 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315006331e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235552768 dispatch_meta_bytes=115016 combine_payload_bytes=235552768 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235667784 ingress_bytes=2349481984 link_bytes=2349481984 token_bytes=471220552 weight_movement_bytes=2113929216 total_nvlink_bytes=2585149768 weight_to_token_ratio=4.486072 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315611839e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=233455616 dispatch_meta_bytes=114312 combine_payload_bytes=233455616 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233569928 ingress_bytes=3052027904 link_bytes=3052027904 token_bytes=467025544 weight_movement_bytes=2818572288 total_nvlink_bytes=3285597832 weight_to_token_ratio=6.035157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709008228e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235175936 dispatch_meta_bytes=114832 combine_payload_bytes=235175936 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235290768 ingress_bytes=3053748224 link_bytes=3053748224 token_bytes=470466704 weight_movement_bytes=2818572288 total_nvlink_bytes=3289038992 weight_to_token_ratio=5.991013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709971536e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28702 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1994, 2033, 2041, 2080, 2057, 2029, 2124, 2026] payload_rows_per_dst=[3988, 4066, 4082, 4160, 4114, 4058, 4248, 4052] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3988, 4066, 4082, 4160, 4114, 4058, 4248, 4052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233570304 dispatch_meta_bytes=114208 combine_payload_bytes=233570304 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233684512 ingress_bytes=4461428736 link_bytes=4461428736 token_bytes=467254816 weight_movement_bytes=4227858432 total_nvlink_bytes=4695113248 weight_to_token_ratio=9.048293 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498213863e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[1, 0, 2, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2012, 2070, 2081, 1997, 2049, 2119, 2101, 1955] payload_rows_per_dst=[4024, 4140, 4162, 3994, 4098, 4238, 4202, 3910] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 4140, 4162, 3994, 4098, 4238, 4202, 3910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2033, 2054, 2013, 2023, 2064, 2088, 2022, 2087] payload_rows_per_dst=[4066, 4108, 4026, 4046, 4128, 4176, 4044, 4174] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4066, 4108, 4026, 4046, 4128, 4176, 4044, 4174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1987, 2073, 2070, 2072, 2002, 2053, 2105, 2022] payload_rows_per_dst=[3974, 4146, 4140, 4144, 4004, 4106, 4210, 4044] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3974, 4146, 4140, 4144, 4004, 4106, 4210, 4044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28558 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2080, 2105, 1960, 1986, 2105, 2041, 2036] payload_rows_per_dst=[4142, 4160, 4210, 3920, 3972, 4210, 4082, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4160, 4210, 3920, 3972, 4210, 4082, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235126784 dispatch_meta_bytes=114808 combine_payload_bytes=235126784 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=235241592 ingress_bytes=4462985216 link_bytes=4462985216 token_bytes=470368376 weight_movement_bytes=4227858432 total_nvlink_bytes=4698226808 weight_to_token_ratio=8.988399 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499085427e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32572 payload_rows_remote=28324 dup_rows=196 dup_rows_remote=150 dedup_ratio=0.005981 router_tokens_per_expert=[2053, 2045, 2072, 2129, 2060, 2011, 2036, 1978] payload_rows_per_dst=[3750, 3818, 4018, 4182, 3934, 4460, 4162, 4248] dup_rows_per_dst=[0, 0, 40, 0, 8, 78, 24, 46] combine_rows_per_src=[3750, 3818, 4018, 4182, 3934, 4460, 4162, 4248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234126936 ingress_bytes=2347941888 link_bytes=2347941888 token_bytes=468139608 weight_movement_bytes=2113929216 total_nvlink_bytes=2582068824 weight_to_token_ratio=4.515596 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314749449e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28578 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2039, 2029, 2095, 2049, 2083, 1985, 2101, 2003] payload_rows_per_dst=[4078, 4058, 4190, 4098, 4166, 3970, 4202, 4006] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4078, 4058, 4190, 4098, 4166, 3970, 4202, 4006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=28588 dup_rows=34 dup_rows_remote=8 dedup_ratio=0.001038 router_tokens_per_expert=[2001, 2049, 2038, 2137, 2019, 2065, 2050, 2025] payload_rows_per_dst=[4146, 4062, 4172, 4170, 4036, 3998, 4100, 4050] dup_rows_per_dst=[26, 0, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 4062, 4172, 4170, 4036, 3998, 4100, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=235405512 ingress_bytes=3758505984 link_bytes=3758505984 token_bytes=470696136 weight_movement_bytes=3523215360 total_nvlink_bytes=3993911496 weight_to_token_ratio=7.485116 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104606463e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235634688 dispatch_meta_bytes=115056 combine_payload_bytes=235634688 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235749744 ingress_bytes=1644920832 link_bytes=1644920832 token_bytes=471384432 weight_movement_bytes=1409286144 total_nvlink_bytes=1880670576 weight_to_token_ratio=2.989675 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210870035e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=233947136 dispatch_meta_bytes=114232 combine_payload_bytes=233947136 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234061368 ingress_bytes=3052519424 link_bytes=3052519424 token_bytes=468008504 weight_movement_bytes=2818572288 total_nvlink_bytes=3286580792 weight_to_token_ratio=6.022481 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709283459e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=232030208 dispatch_meta_bytes=113896 combine_payload_bytes=232030208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232144104 ingress_bytes=1641316352 link_bytes=1641316352 token_bytes=464174312 weight_movement_bytes=1409286144 total_nvlink_bytes=1873460456 weight_to_token_ratio=3.036114 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.190686452e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234110976 dispatch_meta_bytes=114312 combine_payload_bytes=234110976 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234225288 ingress_bytes=2348040192 link_bytes=2348040192 token_bytes=468336264 weight_movement_bytes=2113929216 total_nvlink_bytes=2582265480 weight_to_token_ratio=4.513700 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314804495e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234192896 dispatch_meta_bytes=114384 combine_payload_bytes=234192896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234307280 ingress_bytes=1643479040 link_bytes=1643479040 token_bytes=468500176 weight_movement_bytes=1409286144 total_nvlink_bytes=1877786320 weight_to_token_ratio=3.008080 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202796602e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28486 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2072, 2141, 2050, 1995, 2015, 2007, 2059, 2045] payload_rows_per_dst=[4144, 4282, 4100, 3990, 4030, 4014, 4118, 4090] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4144, 4282, 4100, 3990, 4030, 4014, 4118, 4090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2134, 2027, 2028, 2039, 2104, 1991, 2053, 2008] payload_rows_per_dst=[4268, 4054, 4056, 4078, 4208, 3982, 4106, 4016] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4268, 4054, 4056, 4078, 4208, 3982, 4106, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2032, 1969, 2030, 2057, 2027, 2101, 2113, 2055] payload_rows_per_dst=[4064, 3938, 4060, 4114, 4054, 4202, 4226, 4110] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4064, 3938, 4060, 4114, 4054, 4202, 4226, 4110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28784 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2070, 2116, 2106, 2072, 1992, 1986, 2038, 2004] payload_rows_per_dst=[4140, 4232, 4212, 4144, 3984, 3972, 4076, 4008] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4232, 4212, 4144, 3984, 3972, 4076, 4008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233357312 dispatch_meta_bytes=113944 combine_payload_bytes=233357312 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233471256 ingress_bytes=3051929600 link_bytes=3051929600 token_bytes=466828568 weight_movement_bytes=2818572288 total_nvlink_bytes=3285400856 weight_to_token_ratio=6.037703 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708953182e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32664 payload_rows_remote=28318 dup_rows=104 dup_rows_remote=72 dedup_ratio=0.003174 router_tokens_per_expert=[2018, 2062, 2060, 2023, 2004, 2122, 2036, 2059] payload_rows_per_dst=[4036, 4100, 3908, 4200, 3910, 3820, 4344, 4346] dup_rows_per_dst=[0, 0, 0, 32, 0, 0, 40, 32] combine_rows_per_src=[4036, 4100, 3908, 4200, 3910, 3820, 4344, 4346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28710 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2053, 2042, 1998, 2092, 2017, 2029, 2087, 2066] payload_rows_per_dst=[4106, 4084, 3996, 4184, 4034, 4058, 4174, 4132] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4106, 4084, 3996, 4184, 4034, 4058, 4174, 4132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234913752 ingress_bytes=2348728320 link_bytes=2348728320 token_bytes=469712856 weight_movement_bytes=2113929216 total_nvlink_bytes=2583642072 weight_to_token_ratio=4.500471 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315189818e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234848184 ingress_bytes=3053305856 link_bytes=3053305856 token_bytes=469581752 weight_movement_bytes=2818572288 total_nvlink_bytes=3288154040 weight_to_token_ratio=6.002304 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709723828e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=28448 dup_rows=34 dup_rows_remote=16 dedup_ratio=0.001038 router_tokens_per_expert=[2072, 2088, 2085, 1978, 1975, 2040, 2078, 2068] payload_rows_per_dst=[4286, 4248, 4098, 3946, 4048, 3972, 4000, 4136] dup_rows_per_dst=[18, 6, 0, 0, 10, 0, 0, 0] combine_rows_per_src=[4286, 4248, 4098, 3946, 4048, 3972, 4000, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28524 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2020, 2072, 2122, 1996, 2004, 2085, 2027, 2058] payload_rows_per_dst=[4040, 4144, 4244, 3992, 4008, 4170, 4054, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4040, 4144, 4244, 3992, 4008, 4170, 4054, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235798528 dispatch_meta_bytes=115136 combine_payload_bytes=235798528 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235913664 ingress_bytes=2349727744 link_bytes=2349727744 token_bytes=471712192 weight_movement_bytes=2113929216 total_nvlink_bytes=2585641408 weight_to_token_ratio=4.481396 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315749454e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=231981056 dispatch_meta_bytes=113560 combine_payload_bytes=231981056 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232094616 ingress_bytes=1641267200 link_bytes=1641267200 token_bytes=464075672 weight_movement_bytes=1409286144 total_nvlink_bytes=1873361816 weight_to_token_ratio=3.036759 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.190411221e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235192320 dispatch_meta_bytes=114840 combine_payload_bytes=235192320 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235307160 ingress_bytes=1644478464 link_bytes=1644478464 token_bytes=470499480 weight_movement_bytes=1409286144 total_nvlink_bytes=1879785624 weight_to_token_ratio=2.995298 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208392959e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28590 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2073, 2089, 2085, 1976, 2095, 2053, 1987, 2026] payload_rows_per_dst=[4146, 4178, 4170, 3952, 4190, 4106, 3974, 4052] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 4178, 4170, 3952, 4190, 4106, 3974, 4052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233046016 dispatch_meta_bytes=113856 combine_payload_bytes=233046016 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233159872 ingress_bytes=4460904448 link_bytes=4460904448 token_bytes=466205888 weight_movement_bytes=4227858432 total_nvlink_bytes=4694064320 weight_to_token_ratio=9.068651 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.497920284e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233668608 dispatch_meta_bytes=114096 combine_payload_bytes=233668608 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233782704 ingress_bytes=3052240896 link_bytes=3052240896 token_bytes=467451312 weight_movement_bytes=2818572288 total_nvlink_bytes=3286023600 weight_to_token_ratio=6.029660 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709127495e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28850 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2127, 2029, 2019, 2070, 2087, 1959, 2022] payload_rows_per_dst=[4142, 4254, 4058, 4038, 4140, 4174, 3918, 4044] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4254, 4058, 4038, 4140, 4174, 3918, 4044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2052, 2068, 2057, 2047, 2041, 2058, 1996] payload_rows_per_dst=[4130, 4104, 4136, 4114, 4094, 4082, 4116, 3992] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4104, 4136, 4114, 4094, 4082, 4116, 3992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234209280 dispatch_meta_bytes=114360 combine_payload_bytes=234209280 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234323640 ingress_bytes=3757424640 link_bytes=3757424640 token_bytes=468532920 weight_movement_bytes=3523215360 total_nvlink_bytes=3991748280 weight_to_token_ratio=7.519675 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104000956e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 2054, 2030, 2093, 2087, 2045, 2063, 1981] payload_rows_per_dst=[4062, 4108, 4060, 4186, 4174, 4090, 4126, 3962] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 4108, 4060, 4186, 4174, 4090, 4126, 3962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32646 payload_rows_remote=28394 dup_rows=122 dup_rows_remote=58 dedup_ratio=0.003723 router_tokens_per_expert=[2050, 2090, 2106, 2070, 2015, 2070, 2038, 1945] payload_rows_per_dst=[3720, 3960, 4260, 3986, 4120, 4236, 4112, 4252] dup_rows_per_dst=[0, 4, 14, 0, 18, 16, 6, 64] combine_rows_per_src=[3720, 3960, 4260, 3986, 4120, 4236, 4112, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=234848184 ingress_bytes=3757948928 link_bytes=3757948928 token_bytes=469581752 weight_movement_bytes=3523215360 total_nvlink_bytes=3992797112 weight_to_token_ratio=7.502880 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104294535e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2053, 1992, 2044, 2042, 2128, 2104, 1993, 2028] payload_rows_per_dst=[4106, 3984, 4088, 4084, 4256, 4208, 3986, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4106, 3984, 4088, 4084, 4256, 4208, 3986, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=236339200 dispatch_meta_bytes=115400 combine_payload_bytes=236339200 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236454600 ingress_bytes=1645625344 link_bytes=1645625344 token_bytes=472793800 weight_movement_bytes=1409286144 total_nvlink_bytes=1882079944 weight_to_token_ratio=2.980763 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.214815009e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32712 payload_rows_remote=28566 dup_rows=56 dup_rows_remote=44 dedup_ratio=0.001709 router_tokens_per_expert=[2008, 1999, 2041, 2087, 2107, 2049, 2067, 2026] payload_rows_per_dst=[4146, 3996, 4216, 4174, 4060, 4098, 3982, 4040] dup_rows_per_dst=[12, 26, 18, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 3996, 4216, 4174, 4060, 4098, 3982, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234242048 dispatch_meta_bytes=114376 combine_payload_bytes=234242048 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234356424 ingress_bytes=2348171264 link_bytes=2348171264 token_bytes=468598472 weight_movement_bytes=2113929216 total_nvlink_bytes=2582527688 weight_to_token_ratio=4.511174 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314877890e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28770 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2012, 1999, 2098, 2068, 2033, 2092, 2027] payload_rows_per_dst=[4110, 4024, 3998, 4196, 4136, 4066, 4184, 4054] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4024, 3998, 4196, 4136, 4066, 4184, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232603648 dispatch_meta_bytes=113808 combine_payload_bytes=232603648 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232717456 ingress_bytes=1641889792 link_bytes=1641889792 token_bytes=465321104 weight_movement_bytes=1409286144 total_nvlink_bytes=1874607248 weight_to_token_ratio=3.028631 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.193897477e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2111, 2053, 2055, 2071, 2080, 2020, 1977, 2017] payload_rows_per_dst=[4222, 4106, 4110, 4142, 4160, 4040, 3954, 4034] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4222, 4106, 4110, 4142, 4160, 4040, 3954, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114240 combine_payload_bytes=233963520 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234077760 ingress_bytes=4461821952 link_bytes=4461821952 token_bytes=468041280 weight_movement_bytes=4227858432 total_nvlink_bytes=4695899712 weight_to_token_ratio=9.033089 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498434048e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114440 combine_payload_bytes=234012672 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234127112 ingress_bytes=2347941888 link_bytes=2347941888 token_bytes=468139784 weight_movement_bytes=2113929216 total_nvlink_bytes=2582069000 weight_to_token_ratio=4.515594 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314749449e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235683840 dispatch_meta_bytes=115080 combine_payload_bytes=235683840 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235798920 ingress_bytes=2349613056 link_bytes=2349613056 token_bytes=471482760 weight_movement_bytes=2113929216 total_nvlink_bytes=2585411976 weight_to_token_ratio=4.483577 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315685234e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28612 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2085, 2034, 2078, 2039, 2042, 2032, 2020] payload_rows_per_dst=[4108, 4170, 4068, 4156, 4078, 4084, 4064, 4040] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4170, 4068, 4156, 4078, 4084, 4064, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2002, 2072, 2043, 2100, 1993, 2141, 2058, 1975] payload_rows_per_dst=[4004, 4144, 4086, 4200, 3986, 4282, 4116, 3950] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4004, 4144, 4086, 4200, 3986, 4282, 4116, 3950] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234913752 ingress_bytes=2348728320 link_bytes=2348728320 token_bytes=469712856 weight_movement_bytes=2113929216 total_nvlink_bytes=2583642072 weight_to_token_ratio=4.500471 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315189818e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28664 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2088, 2046, 1984, 2097, 2052, 2002, 2098, 2017] payload_rows_per_dst=[4176, 4092, 3968, 4194, 4104, 4004, 4196, 4034] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4176, 4092, 3968, 4194, 4104, 4004, 4196, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32648 payload_rows_remote=28478 dup_rows=120 dup_rows_remote=98 dedup_ratio=0.003662 router_tokens_per_expert=[2045, 2041, 2059, 2011, 2058, 2055, 2077, 2038] payload_rows_per_dst=[4024, 3340, 4080, 4112, 4134, 4570, 4218, 4170] dup_rows_per_dst=[0, 0, 0, 16, 0, 66, 16, 22] combine_rows_per_src=[4024, 3340, 4080, 4112, 4134, 4570, 4218, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234389504 dispatch_meta_bytes=114448 combine_payload_bytes=234389504 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=234503952 ingress_bytes=939032576 link_bytes=939032576 token_bytes=468893456 weight_movement_bytes=704643072 total_nvlink_bytes=1173536528 weight_to_token_ratio=1.502779 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.258190454e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28566 dup_rows=80 dup_rows_remote=56 dedup_ratio=0.002441 router_tokens_per_expert=[2002, 2112, 2013, 2062, 2093, 1939, 2095, 2068] payload_rows_per_dst=[4122, 4340, 4262, 3912, 4138, 3598, 4190, 4126] dup_rows_per_dst=[24, 26, 30, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4340, 4262, 3912, 4138, 3598, 4190, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234831792 ingress_bytes=1644003328 link_bytes=1644003328 token_bytes=469548976 weight_movement_bytes=1409286144 total_nvlink_bytes=1878835120 weight_to_token_ratio=3.001361 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205732396e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2040, 2049, 1995, 2046, 2035, 2042, 2126] payload_rows_per_dst=[4102, 4080, 4098, 3990, 4092, 4070, 4084, 4252] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4080, 4098, 3990, 4092, 4070, 4084, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28832 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2155, 1968, 2056, 2120, 1968, 2037, 2059] payload_rows_per_dst=[4042, 4310, 3936, 4112, 4240, 3936, 4074, 4118] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4310, 3936, 4112, 4240, 3936, 4074, 4118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234815488 dispatch_meta_bytes=114656 combine_payload_bytes=234815488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234930144 ingress_bytes=1644101632 link_bytes=1644101632 token_bytes=469745632 weight_movement_bytes=1409286144 total_nvlink_bytes=1879031776 weight_to_token_ratio=3.000105 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206282857e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233291776 dispatch_meta_bytes=114304 combine_payload_bytes=233291776 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233406080 ingress_bytes=2347220992 link_bytes=2347220992 token_bytes=466697856 weight_movement_bytes=2113929216 total_nvlink_bytes=2580627072 weight_to_token_ratio=4.529546 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314345778e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2119, 2101, 2081, 2027, 2005, 2035, 2009, 2007] payload_rows_per_dst=[4238, 4202, 4162, 4054, 4010, 4070, 4018, 4014] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4238, 4202, 4162, 4054, 4010, 4070, 4018, 4014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114488 combine_payload_bytes=234012672 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234127160 ingress_bytes=1643298816 link_bytes=1643298816 token_bytes=468139832 weight_movement_bytes=1409286144 total_nvlink_bytes=1877425976 weight_to_token_ratio=3.010396 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201787423e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235208808 ingress_bytes=2349023232 link_bytes=2349023232 token_bytes=470302824 weight_movement_bytes=2113929216 total_nvlink_bytes=2584232040 weight_to_token_ratio=4.494826 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315354957e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=236191744 dispatch_meta_bytes=115328 combine_payload_bytes=236191744 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=236307072 ingress_bytes=2350120960 link_bytes=2350120960 token_bytes=472498816 weight_movement_bytes=2113929216 total_nvlink_bytes=2586428032 weight_to_token_ratio=4.473935 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315969639e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1998, 2050, 2111, 2068, 2014, 2073, 2039, 2031] payload_rows_per_dst=[3996, 4100, 4222, 4136, 4028, 4146, 4078, 4062] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3996, 4100, 4222, 4136, 4028, 4146, 4078, 4062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234126936 ingress_bytes=4461871104 link_bytes=4461871104 token_bytes=468139608 weight_movement_bytes=4227858432 total_nvlink_bytes=4695998040 weight_to_token_ratio=9.031191 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498461571e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28562 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2019, 2125, 2027, 2011, 2041, 1944, 2103, 2114] payload_rows_per_dst=[4038, 4250, 4054, 4022, 4082, 3888, 4206, 4228] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4038, 4250, 4054, 4022, 4082, 3888, 4206, 4228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28736 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2156, 2047, 2058, 2016, 2014, 2008, 2069] payload_rows_per_dst=[4032, 4312, 4094, 4116, 4032, 4028, 4016, 4138] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4312, 4094, 4116, 4032, 4028, 4016, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=28600 dup_rows=60 dup_rows_remote=42 dedup_ratio=0.001831 router_tokens_per_expert=[2072, 2076, 2035, 2056, 2058, 1991, 2095, 2001] payload_rows_per_dst=[4144, 3904, 4070, 4168, 3906, 4016, 4392, 4108] dup_rows_per_dst=[0, 0, 0, 10, 0, 4, 28, 18] combine_rows_per_src=[4144, 3904, 4070, 4168, 3906, 4016, 4392, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234667872 ingress_bytes=3757768704 link_bytes=3757768704 token_bytes=469221216 weight_movement_bytes=3523215360 total_nvlink_bytes=3992436576 weight_to_token_ratio=7.508645 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104193617e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=28502 dup_rows=60 dup_rows_remote=52 dedup_ratio=0.001831 router_tokens_per_expert=[2071, 2060, 2031, 2063, 2035, 2082, 1973, 2069] payload_rows_per_dst=[4206, 4216, 4072, 4116, 3940, 4362, 3946, 3850] dup_rows_per_dst=[8, 24, 0, 0, 0, 28, 0, 0] combine_rows_per_src=[4206, 4216, 4072, 4116, 3940, 4362, 3946, 3850] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233979904 dispatch_meta_bytes=114248 combine_payload_bytes=233979904 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234094152 ingress_bytes=2347909120 link_bytes=2347909120 token_bytes=468074056 weight_movement_bytes=2113929216 total_nvlink_bytes=2582003272 weight_to_token_ratio=4.516228 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314731101e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2081, 2070, 2003, 2066, 2014, 2087, 1999] payload_rows_per_dst=[4128, 4162, 4140, 4006, 4132, 4028, 4174, 3998] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4162, 4140, 4006, 4132, 4028, 4174, 3998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2097, 2052, 2009, 1985, 2011, 2185, 1978, 2067] payload_rows_per_dst=[4194, 4104, 4018, 3970, 4022, 4370, 3956, 4134] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4104, 4018, 3970, 4022, 4370, 3956, 4134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235405312 dispatch_meta_bytes=114944 combine_payload_bytes=235405312 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235520256 ingress_bytes=2349334528 link_bytes=2349334528 token_bytes=470925568 weight_movement_bytes=2113929216 total_nvlink_bytes=2584854784 weight_to_token_ratio=4.488882 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315529270e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234291200 dispatch_meta_bytes=114568 combine_payload_bytes=234291200 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234405768 ingress_bytes=2348220416 link_bytes=2348220416 token_bytes=468696968 weight_movement_bytes=2113929216 total_nvlink_bytes=2582626184 weight_to_token_ratio=4.510226 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314905413e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2003, 2054, 2067, 2040, 2116, 2081, 1957, 2066] payload_rows_per_dst=[4006, 4108, 4134, 4080, 4232, 4162, 3914, 4132] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4006, 4108, 4134, 4080, 4232, 4162, 3914, 4132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233488384 dispatch_meta_bytes=114216 combine_payload_bytes=233488384 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233602600 ingress_bytes=3756703744 link_bytes=3756703744 token_bytes=467090984 weight_movement_bytes=3523215360 total_nvlink_bytes=3990306344 weight_to_token_ratio=7.542889 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103597284e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114960 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235553040 ingress_bytes=1644724224 link_bytes=1644724224 token_bytes=470991120 weight_movement_bytes=1409286144 total_nvlink_bytes=1880277264 weight_to_token_ratio=2.992171 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209769113e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235635000 ingress_bytes=1644806144 link_bytes=1644806144 token_bytes=471155000 weight_movement_bytes=1409286144 total_nvlink_bytes=1880441144 weight_to_token_ratio=2.991131 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210227831e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2104, 2061, 2012, 2038, 2063, 1991, 2039, 2076] payload_rows_per_dst=[4208, 4122, 4024, 4076, 4126, 3982, 4078, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4208, 4122, 4024, 4076, 4126, 3982, 4078, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114640 combine_payload_bytes=234782720 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234897360 ingress_bytes=3053355008 link_bytes=3053355008 token_bytes=469680080 weight_movement_bytes=2818572288 total_nvlink_bytes=3288252368 weight_to_token_ratio=6.001047 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709751351e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7464081192 prefetch_bytes=21843935232 reduce_bytes=16206790656 weight_movement_bytes=38050725888 egress_bytes=3732952872 ingress_bytes=41781854208 link_bytes=41781854208 total_nvlink_bytes=45514807080 weight_to_token_ratio=5.097845 prefetch_share_of_ingress=0.522809 dedup_ratio=0.002155 prefetch_slots=62 unused_prefetch_slots=2 slot_utilization=0.968750 reduce_slots=23 remote_demand=31 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.015625 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.462255894e-04 prefetch_bytes_per_gemm_flop=7.644807449e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28506 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1997, 2078, 1980, 2052, 2090, 2041, 2131, 2015] payload_rows_per_dst=[3994, 4156, 3960, 4104, 4180, 4082, 4262, 4030] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3994, 4156, 3960, 4104, 4180, 4082, 4262, 4030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=28554 dup_rows=66 dup_rows_remote=62 dedup_ratio=0.002014 router_tokens_per_expert=[2067, 2115, 2101, 2003, 1998, 1991, 2046, 2063] payload_rows_per_dst=[3790, 4144, 4242, 4150, 3978, 4042, 4208, 4148] dup_rows_per_dst=[0, 0, 12, 20, 8, 10, 12, 4] combine_rows_per_src=[3790, 4144, 4242, 4150, 3978, 4042, 4208, 4148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28744 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2028, 2016, 2105, 2012, 2121, 1990, 2082] payload_rows_per_dst=[4060, 4056, 4032, 4210, 4024, 4242, 3980, 4164] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4056, 4032, 4210, 4024, 4242, 3980, 4164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235159632 ingress_bytes=2348974080 link_bytes=2348974080 token_bytes=470204496 weight_movement_bytes=2113929216 total_nvlink_bytes=2584133712 weight_to_token_ratio=4.495766 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315327434e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28570 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1999, 1961, 2099, 2050, 2049, 2015, 2109, 2102] payload_rows_per_dst=[3998, 3922, 4198, 4100, 4098, 4030, 4218, 4204] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3998, 3922, 4198, 4100, 4098, 4030, 4218, 4204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233521152 dispatch_meta_bytes=114024 combine_payload_bytes=233521152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233635176 ingress_bytes=1642807296 link_bytes=1642807296 token_bytes=467156328 weight_movement_bytes=1409286144 total_nvlink_bytes=1876442472 weight_to_token_ratio=3.016733 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.199035116e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233914368 dispatch_meta_bytes=114464 combine_payload_bytes=233914368 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234028832 ingress_bytes=3052486656 link_bytes=3052486656 token_bytes=467943200 weight_movement_bytes=2818572288 total_nvlink_bytes=3286515488 weight_to_token_ratio=6.023321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709265110e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28620 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2114, 1987, 2020, 2089, 2119, 2074, 1984, 1997] payload_rows_per_dst=[4228, 3974, 4040, 4178, 4238, 4148, 3968, 3994] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4228, 3974, 4040, 4178, 4238, 4148, 3968, 3994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235470848 dispatch_meta_bytes=114976 combine_payload_bytes=235470848 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235585824 ingress_bytes=3758686208 link_bytes=3758686208 token_bytes=471056672 weight_movement_bytes=3523215360 total_nvlink_bytes=3994272032 weight_to_token_ratio=7.479387 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104707381e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7520420424 prefetch_bytes=17616076800 reduce_bytes=20434649088 weight_movement_bytes=38050725888 egress_bytes=3761128008 ingress_bytes=41810018304 link_bytes=41810018304 total_nvlink_bytes=45571146312 weight_to_token_ratio=5.059654 prefetch_share_of_ingress=0.421336 dedup_ratio=0.000000 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=29 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.463241563e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2081, 2036, 1984, 2039, 2059, 2066, 2007, 2112] payload_rows_per_dst=[4162, 4072, 3968, 4078, 4118, 4132, 4014, 4224] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4162, 4072, 3968, 4078, 4118, 4132, 4014, 4224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7460050736 prefetch_bytes=21139292160 reduce_bytes=14092861440 weight_movement_bytes=35232153600 egress_bytes=3730937648 ingress_bytes=38961266688 link_bytes=38961266688 total_nvlink_bytes=42692204336 weight_to_token_ratio=4.722777 prefetch_share_of_ingress=0.542572 dedup_ratio=0.002495 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=20 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016098 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.363542689e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234045440 dispatch_meta_bytes=114280 combine_payload_bytes=234045440 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234159720 ingress_bytes=2347974656 link_bytes=2347974656 token_bytes=468205160 weight_movement_bytes=2113929216 total_nvlink_bytes=2582134376 weight_to_token_ratio=4.514964 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314767798e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234455040 dispatch_meta_bytes=114480 combine_payload_bytes=234455040 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234569520 ingress_bytes=3757670400 link_bytes=3757670400 token_bytes=469024560 weight_movement_bytes=3523215360 total_nvlink_bytes=3992239920 weight_to_token_ratio=7.511793 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104138571e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235192416 ingress_bytes=3758292992 link_bytes=3758292992 token_bytes=470270048 weight_movement_bytes=3523215360 total_nvlink_bytes=3993485408 weight_to_token_ratio=7.491898 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104487197e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2019, 2023, 2079, 2076, 2147, 2015, 2049, 1976] payload_rows_per_dst=[4038, 4046, 4158, 4152, 4294, 4030, 4098, 3952] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4038, 4046, 4158, 4152, 4294, 4030, 4098, 3952] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7513373584 prefetch_bytes=15502147584 reduce_bytes=26776436736 weight_movement_bytes=42278584320 egress_bytes=3757603728 ingress_bytes=46034354176 link_bytes=46034354176 total_nvlink_bytes=49791957904 weight_to_token_ratio=5.627111 prefetch_share_of_ingress=0.336752 dedup_ratio=0.000000 prefetch_slots=44 unused_prefetch_slots=20 slot_utilization=0.687500 reduce_slots=38 remote_demand=22 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019886 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.611082298e-04 prefetch_bytes_per_gemm_flop=5.425347222e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2065, 2025, 2035, 2048, 2056, 2041, 2040] payload_rows_per_dst=[4148, 4130, 4050, 4070, 4096, 4112, 4082, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4130, 4050, 4070, 4096, 4112, 4082, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7521108720 prefetch_bytes=18320719872 reduce_bytes=25367150592 weight_movement_bytes=43687870464 egress_bytes=3761472240 ingress_bytes=47447506944 link_bytes=47447506944 total_nvlink_bytes=51208979184 weight_to_token_ratio=5.808701 prefetch_share_of_ingress=0.386126 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=36 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.660538958e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234979320 ingress_bytes=1644150784 link_bytes=1644150784 token_bytes=469843960 weight_movement_bytes=1409286144 total_nvlink_bytes=1879130104 weight_to_token_ratio=2.999477 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206558088e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2105, 2046, 2022, 2003, 2055, 2020, 2056, 2077] payload_rows_per_dst=[4210, 4092, 4044, 4006, 4110, 4040, 4112, 4154] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4210, 4092, 4044, 4006, 4110, 4040, 4112, 4154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234995712 ingress_bytes=3053453312 link_bytes=3053453312 token_bytes=469876736 weight_movement_bytes=2818572288 total_nvlink_bytes=3288449024 weight_to_token_ratio=5.998536 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709806397e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7515209040 prefetch_bytes=18320719872 reduce_bytes=11978932224 weight_movement_bytes=30299652096 egress_bytes=3758521680 ingress_bytes=34056339456 link_bytes=34056339456 total_nvlink_bytes=37814861136 weight_to_token_ratio=4.031778 prefetch_share_of_ingress=0.537953 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=17 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.191883032e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235454688 ingress_bytes=1644625920 link_bytes=1644625920 token_bytes=470794464 weight_movement_bytes=1409286144 total_nvlink_bytes=1880080608 weight_to_token_ratio=2.993421 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209218651e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7527663920 prefetch_bytes=18320719872 reduce_bytes=14797504512 weight_movement_bytes=33118224384 egress_bytes=3764750640 ingress_bytes=36881137664 link_bytes=36881137664 total_nvlink_bytes=40645888304 weight_to_token_ratio=4.399535 prefetch_share_of_ingress=0.496750 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=21 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.290743600e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7518453864 prefetch_bytes=19025362944 reduce_bytes=20434649088 weight_movement_bytes=39460012032 egress_bytes=3760144488 ingress_bytes=43218321408 link_bytes=43218321408 total_nvlink_bytes=46978465896 weight_to_token_ratio=5.248421 prefetch_share_of_ingress=0.440215 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=29 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.512528498e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none - [2026-09-16 21:35:53] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 569.1 | throughput per GPU (TFLOP/s/GPU): 585.8 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.418919E+00 | loss scale: 1.0 | grad norm: 5.142 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32730 payload_rows_remote=28538 dup_rows=38 dup_rows_remote=30 dedup_ratio=0.001160 router_tokens_per_expert=[2066, 2014, 2042, 2080, 2057, 2093, 1956, 2076] payload_rows_per_dst=[4192, 4070, 4040, 4318, 4114, 3936, 3912, 4148] dup_rows_per_dst=[8, 8, 2, 20, 0, 0, 0, 0] combine_rows_per_src=[4192, 4070, 4040, 4318, 4114, 3936, 3912, 4148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28810 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2077, 2114, 2058, 1979, 2077, 2021, 2021] payload_rows_per_dst=[4074, 4154, 4228, 4116, 3958, 4154, 4042, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4154, 4228, 4116, 3958, 4154, 4042, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28800 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 2071, 1984, 2060, 1999, 2153, 2039, 2043] payload_rows_per_dst=[4070, 4142, 3968, 4120, 3998, 4306, 4078, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 4142, 3968, 4120, 3998, 4306, 4078, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233783296 dispatch_meta_bytes=114272 combine_payload_bytes=233783296 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233897568 ingress_bytes=3052355584 link_bytes=3052355584 token_bytes=467680864 weight_movement_bytes=2818572288 total_nvlink_bytes=3286253152 weight_to_token_ratio=6.026700 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709191715e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2023, 2122, 2061, 1987, 2092, 2034, 2044, 2021] payload_rows_per_dst=[4046, 4244, 4122, 3974, 4184, 4068, 4088, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4046, 4244, 4122, 3974, 4184, 4068, 4088, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28628 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2058, 2000, 2089, 2063, 2070, 1970, 2085] payload_rows_per_dst=[4098, 4116, 4000, 4178, 4126, 4140, 3940, 4170] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4116, 4000, 4178, 4126, 4140, 3940, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28586 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2067, 1983, 2117, 2091, 1980, 2026, 2091, 2029] payload_rows_per_dst=[4134, 3966, 4234, 4182, 3960, 4052, 4182, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4134, 3966, 4234, 4182, 3960, 4052, 4182, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28674 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2107, 2047, 2143, 1968, 2043, 1981, 2037, 2058] payload_rows_per_dst=[4214, 4094, 4286, 3936, 4086, 3962, 4074, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4214, 4094, 4286, 3936, 4086, 3962, 4074, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=236011520 dispatch_meta_bytes=115240 combine_payload_bytes=236011520 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236126760 ingress_bytes=1645297664 link_bytes=1645297664 token_bytes=472138280 weight_movement_bytes=1409286144 total_nvlink_bytes=1881424424 weight_to_token_ratio=2.984901 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212980137e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=28572 dup_rows=92 dup_rows_remote=90 dedup_ratio=0.002808 router_tokens_per_expert=[2067, 2071, 1992, 2020, 2062, 2059, 2074, 2039] payload_rows_per_dst=[3932, 3974, 3888, 3924, 4302, 4148, 4404, 4104] dup_rows_per_dst=[0, 0, 0, 0, 40, 2, 48, 2] combine_rows_per_src=[3932, 3974, 3888, 3924, 4302, 4148, 4404, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235929600 dispatch_meta_bytes=115200 combine_payload_bytes=235929600 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=236044800 ingress_bytes=4463788032 link_bytes=4463788032 token_bytes=471974400 weight_movement_bytes=4227858432 total_nvlink_bytes=4699832832 weight_to_token_ratio=8.957813 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499534970e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235061280 ingress_bytes=1644232704 link_bytes=1644232704 token_bytes=470007840 weight_movement_bytes=1409286144 total_nvlink_bytes=1879293984 weight_to_token_ratio=2.998431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207016806e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234520576 dispatch_meta_bytes=114512 combine_payload_bytes=234520576 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234635088 ingress_bytes=3053092864 link_bytes=3053092864 token_bytes=469155664 weight_movement_bytes=2818572288 total_nvlink_bytes=3287727952 weight_to_token_ratio=6.007755 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709604561e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234176512 dispatch_meta_bytes=114344 combine_payload_bytes=234176512 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234290856 ingress_bytes=2348105728 link_bytes=2348105728 token_bytes=468467368 weight_movement_bytes=2113929216 total_nvlink_bytes=2582396584 weight_to_token_ratio=4.512436 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314841193e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234897408 dispatch_meta_bytes=114696 combine_payload_bytes=234897408 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235012104 ingress_bytes=2348826624 link_bytes=2348826624 token_bytes=469909512 weight_movement_bytes=2113929216 total_nvlink_bytes=2583838728 weight_to_token_ratio=4.498588 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315244865e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234061824 dispatch_meta_bytes=114648 combine_payload_bytes=234061824 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234176472 ingress_bytes=1643347968 link_bytes=1643347968 token_bytes=468238296 weight_movement_bytes=1409286144 total_nvlink_bytes=1877524440 weight_to_token_ratio=3.009763 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202062654e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=28512 dup_rows=66 dup_rows_remote=60 dedup_ratio=0.002014 router_tokens_per_expert=[2085, 2067, 2018, 2017, 2076, 2071, 1987, 2063] payload_rows_per_dst=[4190, 4164, 4224, 4104, 4234, 3778, 3974, 4034] dup_rows_per_dst=[6, 8, 36, 6, 10, 0, 0, 0] combine_rows_per_src=[4190, 4164, 4224, 4104, 4234, 3778, 3974, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2023, 2084, 2081, 2025, 2063, 2061, 2007, 2040] payload_rows_per_dst=[4046, 4168, 4162, 4050, 4126, 4122, 4014, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4046, 4168, 4162, 4050, 4126, 4122, 4014, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233570304 dispatch_meta_bytes=114288 combine_payload_bytes=233570304 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=233684592 ingress_bytes=938213376 link_bytes=938213376 token_bytes=467254896 weight_movement_bytes=704643072 total_nvlink_bytes=1171897968 weight_to_token_ratio=1.508049 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.253603276e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114568 combine_payload_bytes=234635264 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234749832 ingress_bytes=3053207552 link_bytes=3053207552 token_bytes=469385096 weight_movement_bytes=2818572288 total_nvlink_bytes=3287957384 weight_to_token_ratio=6.004818 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709668782e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28758 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1987, 2036, 2005, 2108, 2046, 2126, 2059, 2017] payload_rows_per_dst=[3974, 4072, 4010, 4216, 4092, 4252, 4118, 4034] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3974, 4072, 4010, 4216, 4092, 4252, 4118, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2012, 2048, 2067, 2057, 2036, 2042, 2007, 2115] payload_rows_per_dst=[4024, 4096, 4134, 4114, 4072, 4084, 4014, 4230] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 4096, 4134, 4114, 4072, 4084, 4014, 4230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28542 dup_rows=62 dup_rows_remote=44 dedup_ratio=0.001892 router_tokens_per_expert=[2095, 2022, 1983, 2019, 2025, 2138, 2071, 2031] payload_rows_per_dst=[4190, 3978, 3966, 4038, 3900, 4126, 4344, 4164] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 38, 18] combine_rows_per_src=[4190, 3978, 3966, 4038, 3900, 4126, 4344, 4164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28602 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 1982, 2012, 2055, 2094, 2077, 2083, 2051] payload_rows_per_dst=[4060, 3964, 4024, 4110, 4188, 4154, 4166, 4102] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 3964, 4024, 4110, 4188, 4154, 4166, 4102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28706 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2043, 2052, 2063, 2095, 2029, 2031, 2070, 2001] payload_rows_per_dst=[4086, 4104, 4126, 4190, 4058, 4062, 4140, 4002] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4086, 4104, 4126, 4190, 4058, 4062, 4140, 4002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28554 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2096, 2107, 2043, 1970, 2044, 2092, 1980, 2052] payload_rows_per_dst=[4192, 4214, 4086, 3940, 4088, 4184, 3960, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4192, 4214, 4086, 3940, 4088, 4184, 3960, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235585536 dispatch_meta_bytes=115032 combine_payload_bytes=235585536 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235700568 ingress_bytes=1644871680 link_bytes=1644871680 token_bytes=471286104 weight_movement_bytes=1409286144 total_nvlink_bytes=1880572248 weight_to_token_ratio=2.990299 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210594805e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32666 payload_rows_remote=28268 dup_rows=102 dup_rows_remote=58 dedup_ratio=0.003113 router_tokens_per_expert=[2030, 2066, 2077, 2038, 2030, 2076, 2046, 2021] payload_rows_per_dst=[4398, 4246, 4216, 4278, 3966, 3660, 3996, 3906] dup_rows_per_dst=[44, 12, 16, 30, 0, 0, 0, 0] combine_rows_per_src=[4398, 4246, 4216, 4278, 3966, 3660, 3996, 3906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234848184 ingress_bytes=1644019712 link_bytes=1644019712 token_bytes=469581752 weight_movement_bytes=1409286144 total_nvlink_bytes=1878867896 weight_to_token_ratio=3.001152 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205824140e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1952, 2041, 2000, 2035, 2073, 2134, 2057, 2092] payload_rows_per_dst=[3904, 4082, 4000, 4070, 4146, 4268, 4114, 4184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3904, 4082, 4000, 4070, 4146, 4268, 4114, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114344 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233930408 ingress_bytes=2347745280 link_bytes=2347745280 token_bytes=467746472 weight_movement_bytes=2113929216 total_nvlink_bytes=2581675688 weight_to_token_ratio=4.519391 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314639357e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234307584 dispatch_meta_bytes=114408 combine_payload_bytes=234307584 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234421992 ingress_bytes=1643593728 link_bytes=1643593728 token_bytes=468729576 weight_movement_bytes=1409286144 total_nvlink_bytes=1878015720 weight_to_token_ratio=3.006608 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203438807e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235159552 dispatch_meta_bytes=114824 combine_payload_bytes=235159552 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=235274376 ingress_bytes=5167661056 link_bytes=5167661056 token_bytes=470433928 weight_movement_bytes=4932501504 total_nvlink_bytes=5402935432 weight_to_token_ratio=10.485004 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893674483e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233914368 dispatch_meta_bytes=114216 combine_payload_bytes=233914368 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234028584 ingress_bytes=2347843584 link_bytes=2347843584 token_bytes=467942952 weight_movement_bytes=2113929216 total_nvlink_bytes=2581872168 weight_to_token_ratio=4.517493 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314694403e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=231571456 dispatch_meta_bytes=113304 combine_payload_bytes=231571456 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231684760 ingress_bytes=1640857600 link_bytes=1640857600 token_bytes=463256216 weight_movement_bytes=1409286144 total_nvlink_bytes=1872542360 weight_to_token_ratio=3.042131 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.188117632e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114488 combine_payload_bytes=234471424 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234585912 ingress_bytes=3053043712 link_bytes=3053043712 token_bytes=469057336 weight_movement_bytes=2818572288 total_nvlink_bytes=3287629624 weight_to_token_ratio=6.009014 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709577038e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28550 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2090, 2109, 2066, 2028, 2020, 2014, 1992] payload_rows_per_dst=[4130, 4180, 4218, 4132, 4056, 4040, 4028, 3984] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4180, 4218, 4132, 4056, 4040, 4028, 3984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28616 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2022, 2052, 2076, 2013, 2078, 2041, 2081] payload_rows_per_dst=[4042, 4044, 4104, 4152, 4026, 4156, 4082, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4044, 4104, 4152, 4026, 4156, 4082, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28538 dup_rows=62 dup_rows_remote=34 dedup_ratio=0.001892 router_tokens_per_expert=[2038, 2005, 2039, 1999, 2078, 2123, 2062, 2040] payload_rows_per_dst=[4076, 4010, 3858, 3998, 4236, 4156, 4204, 4168] dup_rows_per_dst=[0, 0, 0, 0, 22, 0, 12, 28] combine_rows_per_src=[4076, 4010, 3858, 3998, 4236, 4156, 4204, 4168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28616 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1981, 2084, 2062, 2065, 2020, 2040, 2076, 2056] payload_rows_per_dst=[3962, 4168, 4124, 4130, 4040, 4080, 4152, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3962, 4168, 4124, 4130, 4040, 4080, 4152, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2027, 2009, 2050, 2046, 2087, 2088, 2052, 2025] payload_rows_per_dst=[4054, 4018, 4100, 4092, 4174, 4176, 4104, 4050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4054, 4018, 4100, 4092, 4174, 4176, 4104, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233881600 dispatch_meta_bytes=114200 combine_payload_bytes=233881600 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233995800 ingress_bytes=2347810816 link_bytes=2347810816 token_bytes=467877400 weight_movement_bytes=2113929216 total_nvlink_bytes=2581806616 weight_to_token_ratio=4.518126 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314676055e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28536 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2079, 2004, 2066, 1943, 2051, 2116, 2038, 2087] payload_rows_per_dst=[4158, 4008, 4132, 3886, 4102, 4232, 4076, 4174] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4158, 4008, 4132, 3886, 4102, 4232, 4076, 4174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28678 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2097, 2082, 2039, 2001, 2045, 2053, 2040, 2027] payload_rows_per_dst=[4194, 4164, 4078, 4002, 4090, 4106, 4080, 4054] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4164, 4078, 4002, 4090, 4106, 4080, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28472 dup_rows=84 dup_rows_remote=64 dedup_ratio=0.002563 router_tokens_per_expert=[2048, 2052, 2045, 2071, 2064, 2063, 1964, 2077] payload_rows_per_dst=[4212, 4346, 4132, 4174, 4126, 3728, 3858, 4108] dup_rows_per_dst=[20, 42, 8, 14, 0, 0, 0, 0] combine_rows_per_src=[4212, 4346, 4132, 4174, 4126, 3728, 3858, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234422272 dispatch_meta_bytes=114464 combine_payload_bytes=234422272 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234536736 ingress_bytes=1643708416 link_bytes=1643708416 token_bytes=468959008 weight_movement_bytes=1409286144 total_nvlink_bytes=1878245152 weight_to_token_ratio=3.005137 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204081012e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233783296 dispatch_meta_bytes=114288 combine_payload_bytes=233783296 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233897584 ingress_bytes=2347712512 link_bytes=2347712512 token_bytes=467680880 weight_movement_bytes=2113929216 total_nvlink_bytes=2581610096 weight_to_token_ratio=4.520025 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314621008e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234422272 dispatch_meta_bytes=114464 combine_payload_bytes=234422272 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234536736 ingress_bytes=3757637632 link_bytes=3757637632 token_bytes=468959008 weight_movement_bytes=3523215360 total_nvlink_bytes=3992174368 weight_to_token_ratio=7.512843 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104120222e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235635000 ingress_bytes=1644806144 link_bytes=1644806144 token_bytes=471155000 weight_movement_bytes=1409286144 total_nvlink_bytes=1880441144 weight_to_token_ratio=2.991131 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210227831e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233766912 dispatch_meta_bytes=114144 combine_payload_bytes=233766912 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=233881056 ingress_bytes=5166268416 link_bytes=5166268416 token_bytes=467647968 weight_movement_bytes=4932501504 total_nvlink_bytes=5400149472 weight_to_token_ratio=10.547467 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.892894662e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234930176 dispatch_meta_bytes=114712 combine_payload_bytes=234930176 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=235044888 ingress_bytes=3758145536 link_bytes=3758145536 token_bytes=469975064 weight_movement_bytes=3523215360 total_nvlink_bytes=3993190424 weight_to_token_ratio=7.496601 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104404627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233242624 dispatch_meta_bytes=114144 combine_payload_bytes=233242624 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=233356768 ingress_bytes=1642528768 link_bytes=1642528768 token_bytes=466599392 weight_movement_bytes=1409286144 total_nvlink_bytes=1875885536 weight_to_token_ratio=3.020334 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.197475476e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1961, 2073, 2074, 2000, 2048, 2182, 2001, 2045] payload_rows_per_dst=[3922, 4146, 4148, 4000, 4096, 4364, 4002, 4090] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3922, 4146, 4148, 4000, 4096, 4364, 4002, 4090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28602 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 1956, 2083, 2054, 2108, 2021, 2053, 2054] payload_rows_per_dst=[4110, 3912, 4166, 4108, 4216, 4042, 4106, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 3912, 4166, 4108, 4216, 4042, 4106, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32740 payload_rows_remote=28410 dup_rows=28 dup_rows_remote=10 dedup_ratio=0.000854 router_tokens_per_expert=[2021, 1962, 2103, 2038, 2069, 2009, 2101, 2081] payload_rows_per_dst=[4036, 3924, 4206, 4076, 4038, 3860, 4270, 4330] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 10, 18] combine_rows_per_src=[4036, 3924, 4206, 4076, 4038, 3860, 4270, 4330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28532 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2037, 1985, 2040, 2048, 2081, 2118, 2018] payload_rows_per_dst=[4114, 4074, 3970, 4080, 4096, 4162, 4236, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4074, 3970, 4080, 4096, 4162, 4236, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28808 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2121, 1980, 2008, 2053, 2012, 2152, 2015, 2043] payload_rows_per_dst=[4242, 3960, 4016, 4106, 4024, 4304, 4030, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4242, 3960, 4016, 4106, 4024, 4304, 4030, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28482 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2089, 2042, 1997, 2143, 2019, 2067, 1986] payload_rows_per_dst=[4082, 4178, 4084, 3994, 4286, 4038, 4134, 3972] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4178, 4084, 3994, 4286, 4038, 4134, 3972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1959, 2100, 2022, 2104, 2041, 2101, 2088, 1969] payload_rows_per_dst=[3918, 4200, 4044, 4208, 4082, 4202, 4176, 3938] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3918, 4200, 4044, 4208, 4082, 4202, 4176, 3938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28536 dup_rows=62 dup_rows_remote=62 dedup_ratio=0.001892 router_tokens_per_expert=[2082, 2034, 2067, 2033, 2085, 1999, 2035, 2049] payload_rows_per_dst=[4170, 4068, 4128, 4070, 4166, 4162, 4152, 3790] dup_rows_per_dst=[0, 0, 0, 0, 0, 36, 26, 0] combine_rows_per_src=[4170, 4068, 4128, 4070, 4166, 4162, 4152, 3790] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235667456 dispatch_meta_bytes=115072 combine_payload_bytes=235667456 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235782528 ingress_bytes=2349596672 link_bytes=2349596672 token_bytes=471449984 weight_movement_bytes=2113929216 total_nvlink_bytes=2585379200 weight_to_token_ratio=4.483889 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315676059e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234307584 dispatch_meta_bytes=114408 combine_payload_bytes=234307584 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234421992 ingress_bytes=1643593728 link_bytes=1643593728 token_bytes=468729576 weight_movement_bytes=1409286144 total_nvlink_bytes=1878015720 weight_to_token_ratio=3.006608 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203438807e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=232734720 dispatch_meta_bytes=113680 combine_payload_bytes=232734720 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232848400 ingress_bytes=2346663936 link_bytes=2346663936 token_bytes=465583120 weight_movement_bytes=2113929216 total_nvlink_bytes=2579512336 weight_to_token_ratio=4.540391 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314033850e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233734144 dispatch_meta_bytes=114128 combine_payload_bytes=233734144 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233848272 ingress_bytes=2347663360 link_bytes=2347663360 token_bytes=467582416 weight_movement_bytes=2113929216 total_nvlink_bytes=2581511632 weight_to_token_ratio=4.520977 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314593485e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235995136 dispatch_meta_bytes=115232 combine_payload_bytes=235995136 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236110368 ingress_bytes=1645281280 link_bytes=1645281280 token_bytes=472105504 weight_movement_bytes=1409286144 total_nvlink_bytes=1881391648 weight_to_token_ratio=2.985108 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212888394e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234126936 ingress_bytes=4461871104 link_bytes=4461871104 token_bytes=468139608 weight_movement_bytes=4227858432 total_nvlink_bytes=4695998040 weight_to_token_ratio=9.031191 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498461571e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233324544 dispatch_meta_bytes=113928 combine_payload_bytes=233324544 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233438472 ingress_bytes=3051896832 link_bytes=3051896832 token_bytes=466763016 weight_movement_bytes=2818572288 total_nvlink_bytes=3285335304 weight_to_token_ratio=6.038551 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708934833e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233766912 dispatch_meta_bytes=114392 combine_payload_bytes=233766912 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233881304 ingress_bytes=2347696128 link_bytes=2347696128 token_bytes=467648216 weight_movement_bytes=2113929216 total_nvlink_bytes=2581577432 weight_to_token_ratio=4.520341 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314611834e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2012, 2006, 2038, 2025, 2077, 2122, 2053] payload_rows_per_dst=[4102, 4024, 4012, 4076, 4050, 4154, 4244, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4024, 4012, 4076, 4050, 4154, 4244, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32668 payload_rows_remote=28420 dup_rows=100 dup_rows_remote=80 dedup_ratio=0.003052 router_tokens_per_expert=[2027, 2072, 2073, 2044, 1980, 2024, 2096, 2068] payload_rows_per_dst=[4000, 4104, 3920, 4286, 3728, 4174, 4208, 4248] dup_rows_per_dst=[0, 4, 0, 38, 0, 34, 4, 20] combine_rows_per_src=[4000, 4104, 3920, 4286, 3728, 4174, 4208, 4248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2082, 2049, 2092, 2044, 2029, 2011, 1982, 2095] payload_rows_per_dst=[4164, 4098, 4184, 4088, 4058, 4022, 3964, 4190] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4164, 4098, 4184, 4088, 4058, 4022, 3964, 4190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28792 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2122, 2121, 1972, 2004, 2032, 1988, 2079] payload_rows_per_dst=[4132, 4244, 4242, 3944, 4008, 4064, 3976, 4158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4244, 4242, 3944, 4008, 4064, 3976, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28758 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2005, 2034, 2047, 2161, 2035, 1998, 2043] payload_rows_per_dst=[4122, 4010, 4068, 4094, 4322, 4070, 3996, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4010, 4068, 4094, 4322, 4070, 3996, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28754 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1998, 2019, 2065, 2089, 2075, 2007, 2032, 2099] payload_rows_per_dst=[3996, 4038, 4130, 4178, 4150, 4014, 4064, 4198] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3996, 4038, 4130, 4178, 4150, 4014, 4064, 4198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235159632 ingress_bytes=1644331008 link_bytes=1644331008 token_bytes=470204496 weight_movement_bytes=1409286144 total_nvlink_bytes=1879490640 weight_to_token_ratio=2.997177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207567267e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2072, 2070, 2089, 2012, 2009, 2031, 1975, 2126] payload_rows_per_dst=[4144, 4140, 4178, 4024, 4018, 4062, 3950, 4252] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4144, 4140, 4178, 4024, 4018, 4062, 3950, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32696 payload_rows_remote=28530 dup_rows=72 dup_rows_remote=46 dedup_ratio=0.002197 router_tokens_per_expert=[1997, 2038, 2077, 2034, 2028, 2114, 2092, 2004] payload_rows_per_dst=[4166, 4138, 4102, 4112, 4078, 4070, 4184, 3846] dup_rows_per_dst=[26, 16, 0, 8, 22, 0, 0, 0] combine_rows_per_src=[4166, 4138, 4102, 4112, 4078, 4070, 4184, 3846] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=232816640 dispatch_meta_bytes=114000 combine_payload_bytes=232816640 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232930640 ingress_bytes=3051388928 link_bytes=3051388928 token_bytes=465747280 weight_movement_bytes=2818572288 total_nvlink_bytes=3284319568 weight_to_token_ratio=6.051720 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708650428e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235864064 dispatch_meta_bytes=115168 combine_payload_bytes=235864064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235979232 ingress_bytes=1645150208 link_bytes=1645150208 token_bytes=471843296 weight_movement_bytes=1409286144 total_nvlink_bytes=1881129440 weight_to_token_ratio=2.986767 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212154445e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234160128 dispatch_meta_bytes=114336 combine_payload_bytes=234160128 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234274464 ingress_bytes=3052732416 link_bytes=3052732416 token_bytes=468434592 weight_movement_bytes=2818572288 total_nvlink_bytes=3287006880 weight_to_token_ratio=6.017003 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709402726e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235585536 dispatch_meta_bytes=115032 combine_payload_bytes=235585536 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235700568 ingress_bytes=2349514752 link_bytes=2349514752 token_bytes=471286104 weight_movement_bytes=2113929216 total_nvlink_bytes=2585215320 weight_to_token_ratio=4.485448 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315630188e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235552768 dispatch_meta_bytes=115016 combine_payload_bytes=235552768 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235667784 ingress_bytes=1644838912 link_bytes=1644838912 token_bytes=471220552 weight_movement_bytes=1409286144 total_nvlink_bytes=1880506696 weight_to_token_ratio=2.990715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210411318e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235635000 ingress_bytes=2349449216 link_bytes=2349449216 token_bytes=471155000 weight_movement_bytes=2113929216 total_nvlink_bytes=2585084216 weight_to_token_ratio=4.486696 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315593490e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233717760 dispatch_meta_bytes=114304 combine_payload_bytes=233717760 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233832064 ingress_bytes=2347646976 link_bytes=2347646976 token_bytes=467549824 weight_movement_bytes=2113929216 total_nvlink_bytes=2581479040 weight_to_token_ratio=4.521292 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314584311e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2103, 2061, 2052, 2035, 2055, 2001, 2019, 2058] payload_rows_per_dst=[4206, 4122, 4104, 4070, 4110, 4002, 4038, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4206, 4122, 4104, 4070, 4110, 4002, 4038, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=28520 dup_rows=58 dup_rows_remote=46 dedup_ratio=0.001770 router_tokens_per_expert=[2037, 2043, 2035, 2054, 2004, 2051, 2103, 2057] payload_rows_per_dst=[3878, 4072, 4072, 3870, 3998, 4282, 4348, 4190] dup_rows_per_dst=[0, 0, 0, 0, 0, 26, 20, 12] combine_rows_per_src=[3878, 4072, 4072, 3870, 3998, 4282, 4348, 4190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2048, 2027, 2013, 2094, 2099, 1999, 2035, 2069] payload_rows_per_dst=[4096, 4054, 4026, 4188, 4198, 3998, 4070, 4138] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 4054, 4026, 4188, 4198, 3998, 4070, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2036, 2063, 2066, 2011, 1997, 2051, 2115, 2045] payload_rows_per_dst=[4072, 4126, 4132, 4022, 3994, 4102, 4230, 4090] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4072, 4126, 4132, 4022, 3994, 4102, 4230, 4090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235208808 ingress_bytes=2349023232 link_bytes=2349023232 token_bytes=470302824 weight_movement_bytes=2113929216 total_nvlink_bytes=2584232040 weight_to_token_ratio=4.494826 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315354957e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2028, 2040, 2124, 2080, 2063, 1996, 2008] payload_rows_per_dst=[4090, 4056, 4080, 4248, 4160, 4126, 3992, 4016] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4056, 4080, 4248, 4160, 4126, 3992, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2022, 2037, 2113, 2095, 2050, 1968, 2054] payload_rows_per_dst=[4090, 4044, 4074, 4226, 4190, 4100, 3936, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4044, 4074, 4226, 4190, 4100, 3936, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2021, 2058, 2046, 2054, 2087, 2066, 1995] payload_rows_per_dst=[4114, 4042, 4116, 4092, 4108, 4174, 4132, 3990] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4042, 4116, 4092, 4108, 4174, 4132, 3990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32678 payload_rows_remote=28448 dup_rows=90 dup_rows_remote=82 dedup_ratio=0.002747 router_tokens_per_expert=[2093, 2034, 2028, 2103, 2024, 2019, 2038, 2045] payload_rows_per_dst=[4230, 4226, 4334, 4334, 3886, 3808, 3956, 3904] dup_rows_per_dst=[8, 28, 48, 6, 0, 0, 0, 0] combine_rows_per_src=[4230, 4226, 4334, 4334, 3886, 3808, 3956, 3904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233635840 dispatch_meta_bytes=114264 combine_payload_bytes=233635840 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233750104 ingress_bytes=3756851200 link_bytes=3756851200 token_bytes=467385944 weight_movement_bytes=3523215360 total_nvlink_bytes=3990601304 weight_to_token_ratio=7.538129 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103679853e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235208808 ingress_bytes=2349023232 link_bytes=2349023232 token_bytes=470302824 weight_movement_bytes=2113929216 total_nvlink_bytes=2584232040 weight_to_token_ratio=4.494826 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315354957e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114568 combine_payload_bytes=234635264 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234749832 ingress_bytes=2348564480 link_bytes=2348564480 token_bytes=469385096 weight_movement_bytes=2113929216 total_nvlink_bytes=2583314312 weight_to_token_ratio=4.503614 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315098075e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114568 combine_payload_bytes=234635264 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234749832 ingress_bytes=2348564480 link_bytes=2348564480 token_bytes=469385096 weight_movement_bytes=2113929216 total_nvlink_bytes=2583314312 weight_to_token_ratio=4.503614 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315098075e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235176024 ingress_bytes=2348990464 link_bytes=2348990464 token_bytes=470237272 weight_movement_bytes=2113929216 total_nvlink_bytes=2584166488 weight_to_token_ratio=4.495452 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315336608e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114640 combine_payload_bytes=234782720 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234897360 ingress_bytes=2348711936 link_bytes=2348711936 token_bytes=469680080 weight_movement_bytes=2113929216 total_nvlink_bytes=2583609296 weight_to_token_ratio=4.500785 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315180644e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233046016 dispatch_meta_bytes=114120 combine_payload_bytes=233046016 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233160136 ingress_bytes=2346975232 link_bytes=2346975232 token_bytes=466206152 weight_movement_bytes=2113929216 total_nvlink_bytes=2580135368 weight_to_token_ratio=4.534323 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314208162e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28726 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 2103, 1989, 2021, 2099, 2064, 2013, 2064] payload_rows_per_dst=[4062, 4206, 3978, 4042, 4198, 4128, 4026, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 4206, 3978, 4042, 4198, 4128, 4026, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32722 payload_rows_remote=28550 dup_rows=46 dup_rows_remote=28 dedup_ratio=0.001404 router_tokens_per_expert=[2059, 2022, 2037, 2072, 2047, 2048, 2061, 2038] payload_rows_per_dst=[3934, 4044, 4074, 4144, 4256, 3946, 4152, 4172] dup_rows_per_dst=[0, 0, 0, 0, 20, 0, 8, 18] combine_rows_per_src=[3934, 4044, 4074, 4144, 4256, 3946, 4152, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235323392 dispatch_meta_bytes=114904 combine_payload_bytes=235323392 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=235438296 ingress_bytes=939966464 link_bytes=939966464 token_bytes=470761688 weight_movement_bytes=704643072 total_nvlink_bytes=1175404760 weight_to_token_ratio=1.496815 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.263419837e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28530 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2012, 2019, 1997, 2020, 2050, 2119, 2121] payload_rows_per_dst=[4092, 4024, 4038, 3994, 4040, 4100, 4238, 4242] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4024, 4038, 3994, 4040, 4100, 4238, 4242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2057, 2041, 2022, 2067, 2094, 2038, 2028] payload_rows_per_dst=[4074, 4114, 4082, 4044, 4134, 4188, 4076, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4114, 4082, 4044, 4134, 4188, 4076, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2090, 2035, 2006, 2004, 2086, 2024, 2065] payload_rows_per_dst=[4148, 4180, 4070, 4012, 4008, 4172, 4048, 4130] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4180, 4070, 4012, 4008, 4172, 4048, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2090, 2062, 2006, 2075, 2027, 2059, 2013, 2052] payload_rows_per_dst=[4180, 4124, 4012, 4150, 4054, 4118, 4026, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4180, 4124, 4012, 4150, 4054, 4118, 4026, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2053, 1952, 2014, 2050, 2059, 2126, 2066, 2064] payload_rows_per_dst=[4106, 3904, 4028, 4100, 4118, 4252, 4132, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4106, 3904, 4028, 4100, 4118, 4252, 4132, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32748 payload_rows_remote=28708 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000610 router_tokens_per_expert=[2010, 2069, 2053, 2047, 2016, 2070, 2062, 2057] payload_rows_per_dst=[4040, 4218, 4106, 4108, 3948, 4092, 4122, 4114] dup_rows_per_dst=[0, 18, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[4040, 4218, 4106, 4108, 3948, 4092, 4122, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233881600 dispatch_meta_bytes=114312 combine_payload_bytes=233881600 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233995912 ingress_bytes=2347810816 link_bytes=2347810816 token_bytes=467877512 weight_movement_bytes=2113929216 total_nvlink_bytes=2581806728 weight_to_token_ratio=4.518125 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314676055e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233717760 dispatch_meta_bytes=114120 combine_payload_bytes=233717760 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233831880 ingress_bytes=2347646976 link_bytes=2347646976 token_bytes=467549640 weight_movement_bytes=2113929216 total_nvlink_bytes=2581478856 weight_to_token_ratio=4.521294 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314584311e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234127360 dispatch_meta_bytes=114320 combine_payload_bytes=234127360 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234241680 ingress_bytes=3757342720 link_bytes=3757342720 token_bytes=468369040 weight_movement_bytes=3523215360 total_nvlink_bytes=3991584400 weight_to_token_ratio=7.522306 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103955084e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235225088 dispatch_meta_bytes=114856 combine_payload_bytes=235225088 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235339944 ingress_bytes=3053797376 link_bytes=3053797376 token_bytes=470565032 weight_movement_bytes=2818572288 total_nvlink_bytes=3289137320 weight_to_token_ratio=5.989761 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709999059e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234192896 dispatch_meta_bytes=114352 combine_payload_bytes=234192896 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234307248 ingress_bytes=1643479040 link_bytes=1643479040 token_bytes=468500144 weight_movement_bytes=1409286144 total_nvlink_bytes=1877786288 weight_to_token_ratio=3.008080 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202796602e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114960 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235553040 ingress_bytes=1644724224 link_bytes=1644724224 token_bytes=470991120 weight_movement_bytes=1409286144 total_nvlink_bytes=1880277264 weight_to_token_ratio=2.992171 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209769113e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28568 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2077, 2031, 2059, 2100, 1921, 2024, 2098, 2074] payload_rows_per_dst=[4154, 4062, 4118, 4200, 3842, 4048, 4196, 4148] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4154, 4062, 4118, 4200, 3842, 4048, 4196, 4148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235175936 dispatch_meta_bytes=114912 combine_payload_bytes=235175936 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235290848 ingress_bytes=3053748224 link_bytes=3053748224 token_bytes=470466784 weight_movement_bytes=2818572288 total_nvlink_bytes=3289039072 weight_to_token_ratio=5.991012 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709971536e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234029056 dispatch_meta_bytes=114272 combine_payload_bytes=234029056 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234143328 ingress_bytes=2347958272 link_bytes=2347958272 token_bytes=468172384 weight_movement_bytes=2113929216 total_nvlink_bytes=2582101600 weight_to_token_ratio=4.515280 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314758624e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32640 payload_rows_remote=28248 dup_rows=128 dup_rows_remote=56 dedup_ratio=0.003906 router_tokens_per_expert=[2088, 2062, 1997, 2023, 2052, 2083, 2093, 1986] payload_rows_per_dst=[3928, 3704, 4212, 4026, 4054, 4218, 4106, 4392] dup_rows_per_dst=[0, 0, 16, 14, 0, 10, 16, 72] combine_rows_per_src=[3928, 3704, 4212, 4026, 4054, 4218, 4106, 4392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2086, 2052, 2050, 2078, 2009, 2046, 1989] payload_rows_per_dst=[4148, 4172, 4104, 4100, 4156, 4018, 4092, 3978] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4172, 4104, 4100, 4156, 4018, 4092, 3978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28726 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2110, 2063, 2040, 2085, 2021, 2056, 1950] payload_rows_per_dst=[4118, 4220, 4126, 4080, 4170, 4042, 4112, 3900] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4220, 4126, 4080, 4170, 4042, 4112, 3900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28792 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2025, 2107, 2097, 2057, 1988, 2070, 2001, 2039] payload_rows_per_dst=[4050, 4214, 4194, 4114, 3976, 4140, 4002, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4050, 4214, 4194, 4114, 3976, 4140, 4002, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2026, 2023, 2007, 2011, 2157, 2068, 2055, 2037] payload_rows_per_dst=[4052, 4046, 4014, 4022, 4314, 4136, 4110, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4052, 4046, 4014, 4022, 4314, 4136, 4110, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28708 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2102, 2030, 2033, 2115, 2043, 1994, 1993] payload_rows_per_dst=[4148, 4204, 4060, 4066, 4230, 4086, 3988, 3986] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4204, 4060, 4066, 4230, 4086, 3988, 3986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=28622 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.001587 router_tokens_per_expert=[2045, 2068, 2082, 2067, 2003, 1986, 2096, 2037] payload_rows_per_dst=[4094, 4208, 4210, 4080, 4102, 3912, 4036, 4074] dup_rows_per_dst=[0, 18, 14, 0, 20, 0, 0, 0] combine_rows_per_src=[4094, 4208, 4210, 4080, 4102, 3912, 4036, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=231407616 dispatch_meta_bytes=113216 combine_payload_bytes=231407616 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231520832 ingress_bytes=1640693760 link_bytes=1640693760 token_bytes=462928448 weight_movement_bytes=1409286144 total_nvlink_bytes=1872214592 weight_to_token_ratio=3.044285 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.187200197e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234913792 dispatch_meta_bytes=114704 combine_payload_bytes=234913792 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235028496 ingress_bytes=2348843008 link_bytes=2348843008 token_bytes=469942288 weight_movement_bytes=2113929216 total_nvlink_bytes=2583871504 weight_to_token_ratio=4.498274 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315254039e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235323392 dispatch_meta_bytes=114904 combine_payload_bytes=235323392 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235438296 ingress_bytes=3053895680 link_bytes=3053895680 token_bytes=470761688 weight_movement_bytes=2818572288 total_nvlink_bytes=3289333976 weight_to_token_ratio=5.987259 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710054105e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235864064 dispatch_meta_bytes=115168 combine_payload_bytes=235864064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235979232 ingress_bytes=1645150208 link_bytes=1645150208 token_bytes=471843296 weight_movement_bytes=1409286144 total_nvlink_bytes=1881129440 weight_to_token_ratio=2.986767 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212154445e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2106, 2013, 2096, 2060, 2046, 1971, 2076, 2016] payload_rows_per_dst=[4212, 4026, 4192, 4120, 4092, 3942, 4152, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4212, 4026, 4192, 4120, 4092, 3942, 4152, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235405512 ingress_bytes=3053862912 link_bytes=3053862912 token_bytes=470696136 weight_movement_bytes=2818572288 total_nvlink_bytes=3289268424 weight_to_token_ratio=5.988093 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710035756e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235175936 dispatch_meta_bytes=114832 combine_payload_bytes=235175936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235290768 ingress_bytes=1644462080 link_bytes=1644462080 token_bytes=470466704 weight_movement_bytes=1409286144 total_nvlink_bytes=1879752848 weight_to_token_ratio=2.995507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208301216e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114696 combine_payload_bytes=234471424 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234586120 ingress_bytes=4462329856 link_bytes=4462329856 token_bytes=469057544 weight_movement_bytes=4227858432 total_nvlink_bytes=4696915976 weight_to_token_ratio=9.013518 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498718453e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234799008 ingress_bytes=3053256704 link_bytes=3053256704 token_bytes=469483424 weight_movement_bytes=2818572288 total_nvlink_bytes=3288055712 weight_to_token_ratio=6.003561 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709696305e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32650 payload_rows_remote=28464 dup_rows=118 dup_rows_remote=84 dedup_ratio=0.003601 router_tokens_per_expert=[1993, 2075, 2078, 2050, 2013, 2145, 2060, 1970] payload_rows_per_dst=[3820, 4156, 3680, 3954, 4140, 4550, 4164, 4186] dup_rows_per_dst=[0, 0, 0, 0, 32, 52, 0, 34] combine_rows_per_src=[3820, 4156, 3680, 3954, 4140, 4550, 4164, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28640 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2051, 2055, 2040, 2036, 2036, 2064, 2022] payload_rows_per_dst=[4160, 4102, 4110, 4080, 4072, 4072, 4128, 4044] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4102, 4110, 4080, 4072, 4072, 4128, 4044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28842 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2079, 1971, 2071, 2090, 2061, 1963, 2023, 2126] payload_rows_per_dst=[4158, 3942, 4142, 4180, 4122, 3926, 4046, 4252] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4158, 3942, 4142, 4180, 4122, 3926, 4046, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2092, 2060, 2087, 1976, 2027, 2048, 2014, 2080] payload_rows_per_dst=[4184, 4120, 4174, 3952, 4054, 4096, 4028, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4184, 4120, 4174, 3952, 4054, 4096, 4028, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2042, 2027, 2092, 2071, 2064, 1993, 2030] payload_rows_per_dst=[4130, 4084, 4054, 4184, 4142, 4128, 3986, 4060] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4084, 4054, 4184, 4142, 4128, 3986, 4060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2072, 2009, 2086, 2028, 2035, 2023, 2127, 2004] payload_rows_per_dst=[4144, 4018, 4172, 4056, 4070, 4046, 4254, 4008] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4144, 4018, 4172, 4056, 4070, 4046, 4254, 4008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32664 payload_rows_remote=28170 dup_rows=104 dup_rows_remote=24 dedup_ratio=0.003174 router_tokens_per_expert=[2077, 1980, 2020, 2048, 2100, 2054, 2050, 2055] payload_rows_per_dst=[4494, 4080, 3976, 4108, 3834, 3972, 4100, 4100] dup_rows_per_dst=[80, 14, 4, 6, 0, 0, 0, 0] combine_rows_per_src=[4494, 4080, 3976, 4108, 3834, 3972, 4100, 4100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233177088 dispatch_meta_bytes=114192 combine_payload_bytes=233177088 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233291280 ingress_bytes=1642463232 link_bytes=1642463232 token_bytes=466468368 weight_movement_bytes=1409286144 total_nvlink_bytes=1875754512 weight_to_token_ratio=3.021183 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.197108501e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234618880 dispatch_meta_bytes=114560 combine_payload_bytes=234618880 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234733440 ingress_bytes=3053191168 link_bytes=3053191168 token_bytes=469352320 weight_movement_bytes=2818572288 total_nvlink_bytes=3287924608 weight_to_token_ratio=6.005238 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709659608e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235225088 dispatch_meta_bytes=114856 combine_payload_bytes=235225088 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235339944 ingress_bytes=3053797376 link_bytes=3053797376 token_bytes=470565032 weight_movement_bytes=2818572288 total_nvlink_bytes=3289137320 weight_to_token_ratio=5.989761 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709999059e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28720 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1997, 2069, 2104, 2024, 2063, 2058, 1991, 2078] payload_rows_per_dst=[3994, 4138, 4208, 4048, 4126, 4116, 3982, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3994, 4138, 4208, 4048, 4126, 4116, 3982, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=236273664 dispatch_meta_bytes=115368 combine_payload_bytes=236273664 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236389032 ingress_bytes=2350202880 link_bytes=2350202880 token_bytes=472662696 weight_movement_bytes=2113929216 total_nvlink_bytes=2586591912 weight_to_token_ratio=4.472384 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.316015511e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235225088 dispatch_meta_bytes=114856 combine_payload_bytes=235225088 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235339944 ingress_bytes=3758440448 link_bytes=3758440448 token_bytes=470565032 weight_movement_bytes=3523215360 total_nvlink_bytes=3993780392 weight_to_token_ratio=7.487202 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104569766e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235635000 ingress_bytes=1644806144 link_bytes=1644806144 token_bytes=471155000 weight_movement_bytes=1409286144 total_nvlink_bytes=1880441144 weight_to_token_ratio=2.991131 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210227831e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=230768640 dispatch_meta_bytes=112776 combine_payload_bytes=230768640 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=230881416 ingress_bytes=2344697856 link_bytes=2344697856 token_bytes=461650056 weight_movement_bytes=2113929216 total_nvlink_bytes=2575579272 weight_to_token_ratio=4.579073 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.312932927e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235274240 dispatch_meta_bytes=114880 combine_payload_bytes=235274240 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235389120 ingress_bytes=2349203456 link_bytes=2349203456 token_bytes=470663360 weight_movement_bytes=2113929216 total_nvlink_bytes=2584592576 weight_to_token_ratio=4.491383 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315455875e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32678 payload_rows_remote=28642 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.002747 router_tokens_per_expert=[2024, 2039, 2087, 2063, 2058, 2073, 2022, 2018] payload_rows_per_dst=[3718, 3884, 4050, 4172, 4164, 4146, 4508, 4036] dup_rows_per_dst=[0, 0, 0, 12, 18, 0, 60, 0] combine_rows_per_src=[3718, 3884, 4050, 4172, 4164, 4146, 4508, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2021, 2066, 2033, 2038, 2049, 2096, 2040] payload_rows_per_dst=[4082, 4042, 4132, 4066, 4076, 4098, 4192, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4042, 4132, 4066, 4076, 4098, 4192, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2076, 2066, 2070, 2043, 2049, 2076, 2008, 1996] payload_rows_per_dst=[4152, 4132, 4140, 4086, 4098, 4152, 4016, 3992] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4152, 4132, 4140, 4086, 4098, 4152, 4016, 3992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28724 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1998, 2120, 2058, 2046, 2111, 2022, 2016, 2013] payload_rows_per_dst=[3996, 4240, 4116, 4092, 4222, 4044, 4032, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3996, 4240, 4116, 4092, 4222, 4044, 4032, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28770 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2069, 1999, 2061, 2049, 2057, 2059, 2046] payload_rows_per_dst=[4088, 4138, 3998, 4122, 4098, 4114, 4118, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4138, 3998, 4122, 4098, 4114, 4118, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28746 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 2011, 2051, 2085, 2077, 2064, 1962, 2103] payload_rows_per_dst=[4062, 4022, 4102, 4170, 4154, 4128, 3924, 4206] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 4022, 4102, 4170, 4154, 4128, 3924, 4206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28502 dup_rows=84 dup_rows_remote=66 dedup_ratio=0.002563 router_tokens_per_expert=[2016, 2077, 2034, 2058, 2033, 2067, 2080, 2019] payload_rows_per_dst=[4182, 4402, 4032, 4124, 4122, 3902, 3948, 3972] dup_rows_per_dst=[18, 50, 6, 0, 10, 0, 0, 0] combine_rows_per_src=[4182, 4402, 4032, 4124, 4122, 3902, 3948, 3972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235159632 ingress_bytes=3053617152 link_bytes=3053617152 token_bytes=470204496 weight_movement_bytes=2818572288 total_nvlink_bytes=3288776784 weight_to_token_ratio=5.994354 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709898141e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114928 combine_payload_bytes=234635264 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=234750192 ingress_bytes=1643921408 link_bytes=1643921408 token_bytes=469385456 weight_movement_bytes=1409286144 total_nvlink_bytes=1878671600 weight_to_token_ratio=3.002407 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205273678e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28744 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2085, 2055, 2107, 2012, 1977, 1968, 2091, 2089] payload_rows_per_dst=[4170, 4110, 4214, 4024, 3954, 3936, 4182, 4178] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4170, 4110, 4214, 4024, 3954, 3936, 4182, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235651392 ingress_bytes=1644822528 link_bytes=1644822528 token_bytes=471187776 weight_movement_bytes=1409286144 total_nvlink_bytes=1880473920 weight_to_token_ratio=2.990923 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210319574e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235307008 dispatch_meta_bytes=114896 combine_payload_bytes=235307008 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235421904 ingress_bytes=3758522368 link_bytes=3758522368 token_bytes=470728912 weight_movement_bytes=3523215360 total_nvlink_bytes=3993944272 weight_to_token_ratio=7.484595 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104615638e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235683840 dispatch_meta_bytes=115080 combine_payload_bytes=235683840 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235798920 ingress_bytes=3054256128 link_bytes=3054256128 token_bytes=471482760 weight_movement_bytes=2818572288 total_nvlink_bytes=3290055048 weight_to_token_ratio=5.978103 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710255941e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235487232 dispatch_meta_bytes=114984 combine_payload_bytes=235487232 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235602216 ingress_bytes=2349416448 link_bytes=2349416448 token_bytes=471089448 weight_movement_bytes=2113929216 total_nvlink_bytes=2585018664 weight_to_token_ratio=4.487320 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315575141e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233488384 dispatch_meta_bytes=114272 combine_payload_bytes=233488384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233602656 ingress_bytes=1642774528 link_bytes=1642774528 token_bytes=467091040 weight_movement_bytes=1409286144 total_nvlink_bytes=1876377184 weight_to_token_ratio=3.017155 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.198851629e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235470848 dispatch_meta_bytes=114976 combine_payload_bytes=235470848 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235585824 ingress_bytes=2349400064 link_bytes=2349400064 token_bytes=471056672 weight_movement_bytes=2113929216 total_nvlink_bytes=2584985888 weight_to_token_ratio=4.487632 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315565967e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2072, 2016, 1957, 2063, 2073, 2027, 2085] payload_rows_per_dst=[4182, 4144, 4032, 3914, 4126, 4146, 4054, 4170] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4144, 4032, 3914, 4126, 4146, 4054, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32730 payload_rows_remote=28524 dup_rows=38 dup_rows_remote=26 dedup_ratio=0.001160 router_tokens_per_expert=[2088, 2036, 2032, 2012, 2046, 2090, 2015, 2065] payload_rows_per_dst=[4176, 4072, 3800, 4056, 3986, 4306, 4128, 4206] dup_rows_per_dst=[0, 0, 0, 8, 0, 10, 8, 12] combine_rows_per_src=[4176, 4072, 3800, 4056, 3986, 4306, 4128, 4206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28702 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1972, 2006, 2115, 2036, 2073, 2100, 2033, 2049] payload_rows_per_dst=[3944, 4012, 4230, 4072, 4146, 4200, 4066, 4098] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3944, 4012, 4230, 4072, 4146, 4200, 4066, 4098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2043, 2026, 2069, 2087, 2098, 2025, 2002] payload_rows_per_dst=[4068, 4086, 4052, 4138, 4174, 4196, 4050, 4004] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4086, 4052, 4138, 4174, 4196, 4050, 4004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 1981, 2074, 2059, 2080, 2019, 2038, 2102] payload_rows_per_dst=[4062, 3962, 4148, 4118, 4160, 4038, 4076, 4204] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 3962, 4148, 4118, 4160, 4038, 4076, 4204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114568 combine_payload_bytes=234635264 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234749832 ingress_bytes=3053207552 link_bytes=3053207552 token_bytes=469385096 weight_movement_bytes=2818572288 total_nvlink_bytes=3287957384 weight_to_token_ratio=6.004818 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709668782e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28736 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2033, 2016, 2077, 2081, 2070, 2041, 2059, 2007] payload_rows_per_dst=[4066, 4032, 4154, 4162, 4140, 4082, 4118, 4014] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4066, 4032, 4154, 4162, 4140, 4082, 4118, 4014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32712 payload_rows_remote=28562 dup_rows=56 dup_rows_remote=40 dedup_ratio=0.001709 router_tokens_per_expert=[2049, 2084, 1998, 2078, 2104, 1978, 2022, 2071] payload_rows_per_dst=[4150, 4132, 3996, 4156, 4308, 3846, 4220, 3904] dup_rows_per_dst=[16, 0, 0, 0, 10, 0, 30, 0] combine_rows_per_src=[4150, 4132, 3996, 4156, 4308, 3846, 4220, 3904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233668608 dispatch_meta_bytes=114200 combine_payload_bytes=233668608 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233782808 ingress_bytes=2347597824 link_bytes=2347597824 token_bytes=467451416 weight_movement_bytes=2113929216 total_nvlink_bytes=2581380632 weight_to_token_ratio=4.522244 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314556788e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28678 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1967, 2045, 2096, 2045, 2102, 2052, 2025, 2052] payload_rows_per_dst=[3934, 4090, 4192, 4090, 4204, 4104, 4050, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3934, 4090, 4192, 4090, 4204, 4104, 4050, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235126784 dispatch_meta_bytes=114808 combine_payload_bytes=235126784 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235241592 ingress_bytes=3053699072 link_bytes=3053699072 token_bytes=470368376 weight_movement_bytes=2818572288 total_nvlink_bytes=3288940664 weight_to_token_ratio=5.992266 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709944013e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=235356336 ingress_bytes=5167742976 link_bytes=5167742976 token_bytes=470597808 weight_movement_bytes=4932501504 total_nvlink_bytes=5403099312 weight_to_token_ratio=10.481352 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893720354e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=114920 combine_payload_bytes=235356160 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235471080 ingress_bytes=2349285376 link_bytes=2349285376 token_bytes=470827240 weight_movement_bytes=2113929216 total_nvlink_bytes=2584756456 weight_to_token_ratio=4.489819 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315501747e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235405312 dispatch_meta_bytes=114944 combine_payload_bytes=235405312 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235520256 ingress_bytes=1644691456 link_bytes=1644691456 token_bytes=470925568 weight_movement_bytes=1409286144 total_nvlink_bytes=1880211712 weight_to_token_ratio=2.992588 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209585626e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233979904 dispatch_meta_bytes=114408 combine_payload_bytes=233979904 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234094312 ingress_bytes=3052552192 link_bytes=3052552192 token_bytes=468074216 weight_movement_bytes=2818572288 total_nvlink_bytes=3286646504 weight_to_token_ratio=6.021635 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709301808e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234930176 dispatch_meta_bytes=114712 combine_payload_bytes=234930176 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235044888 ingress_bytes=2348859392 link_bytes=2348859392 token_bytes=469975064 weight_movement_bytes=2113929216 total_nvlink_bytes=2583904280 weight_to_token_ratio=4.497960 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315263213e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28782 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 2045, 2170, 2003, 1993, 2066, 2007, 2053] payload_rows_per_dst=[4094, 4090, 4340, 4006, 3986, 4132, 4014, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 4090, 4340, 4006, 3986, 4132, 4014, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32658 payload_rows_remote=28470 dup_rows=110 dup_rows_remote=82 dedup_ratio=0.003357 router_tokens_per_expert=[2051, 2058, 2102, 2015, 1993, 2008, 2095, 2062] payload_rows_per_dst=[3906, 4084, 4196, 4118, 3724, 4100, 4342, 4188] dup_rows_per_dst=[0, 0, 20, 24, 0, 14, 24, 28] combine_rows_per_src=[3906, 4084, 4196, 4118, 3724, 4100, 4342, 4188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2109, 2109, 2025, 2016, 2046, 2099, 2032, 1948] payload_rows_per_dst=[4218, 4218, 4050, 4032, 4092, 4198, 4064, 3896] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4218, 4218, 4050, 4032, 4092, 4198, 4064, 3896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235782144 dispatch_meta_bytes=115128 combine_payload_bytes=235782144 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235897272 ingress_bytes=1645068288 link_bytes=1645068288 token_bytes=471679416 weight_movement_bytes=1409286144 total_nvlink_bytes=1880965560 weight_to_token_ratio=2.987805 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211695728e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2017, 2032, 2082, 2098, 2067, 2038, 1960, 2090] payload_rows_per_dst=[4034, 4064, 4164, 4196, 4134, 4076, 3920, 4180] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4034, 4064, 4164, 4196, 4134, 4076, 3920, 4180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2002, 2082, 2077, 2061, 2057, 1969, 2075] payload_rows_per_dst=[4122, 4004, 4164, 4154, 4122, 4114, 3938, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4004, 4164, 4154, 4122, 4114, 3938, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28598 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2103, 2016, 1977, 2042, 2024, 2085, 2063, 2074] payload_rows_per_dst=[4206, 4032, 3954, 4084, 4048, 4170, 4126, 4148] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4206, 4032, 3954, 4084, 4048, 4170, 4126, 4148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2005, 2030, 2022, 2087, 2023, 2124, 2063, 2030] payload_rows_per_dst=[4010, 4060, 4044, 4174, 4046, 4248, 4126, 4060] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4010, 4060, 4044, 4174, 4046, 4248, 4126, 4060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=28364 dup_rows=48 dup_rows_remote=26 dedup_ratio=0.001465 router_tokens_per_expert=[2073, 2062, 2022, 2040, 2022, 2156, 1979, 2030] payload_rows_per_dst=[4356, 4032, 3998, 4070, 4250, 4078, 3876, 4060] dup_rows_per_dst=[22, 0, 0, 0, 26, 0, 0, 0] combine_rows_per_src=[4356, 4032, 3998, 4070, 4250, 4078, 3876, 4060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233226240 dispatch_meta_bytes=114208 combine_payload_bytes=233226240 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233340448 ingress_bytes=3051798528 link_bytes=3051798528 token_bytes=466566688 weight_movement_bytes=2818572288 total_nvlink_bytes=3285138976 weight_to_token_ratio=6.041092 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708879787e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235257984 ingress_bytes=1644429312 link_bytes=1644429312 token_bytes=470401152 weight_movement_bytes=1409286144 total_nvlink_bytes=1879687296 weight_to_token_ratio=2.995924 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208117729e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234438384 ingress_bytes=2348253184 link_bytes=2348253184 token_bytes=468762352 weight_movement_bytes=2113929216 total_nvlink_bytes=2582691568 weight_to_token_ratio=4.509597 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314923762e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235634688 dispatch_meta_bytes=115056 combine_payload_bytes=235634688 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235749744 ingress_bytes=3758850048 link_bytes=3758850048 token_bytes=471384432 weight_movement_bytes=3523215360 total_nvlink_bytes=3994599792 weight_to_token_ratio=7.474187 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104799125e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2027, 2076, 1990, 2096, 2110, 2055, 2000] payload_rows_per_dst=[4060, 4054, 4152, 3980, 4192, 4220, 4110, 4000] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4054, 4152, 3980, 4192, 4220, 4110, 4000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114392 combine_payload_bytes=234274816 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234389208 ingress_bytes=2348204032 link_bytes=2348204032 token_bytes=468664024 weight_movement_bytes=2113929216 total_nvlink_bytes=2582593240 weight_to_token_ratio=4.510543 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314896239e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234242048 dispatch_meta_bytes=114376 combine_payload_bytes=234242048 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234356424 ingress_bytes=3757457408 link_bytes=3757457408 token_bytes=468598472 weight_movement_bytes=3523215360 total_nvlink_bytes=3991813832 weight_to_token_ratio=7.518623 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104019304e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232357888 dispatch_meta_bytes=113560 combine_payload_bytes=232357888 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232471448 ingress_bytes=3050930176 link_bytes=3050930176 token_bytes=464829336 weight_movement_bytes=2818572288 total_nvlink_bytes=3283401624 weight_to_token_ratio=6.063671 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708393546e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234094592 dispatch_meta_bytes=114304 combine_payload_bytes=234094592 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234208896 ingress_bytes=1643380736 link_bytes=1643380736 token_bytes=468303488 weight_movement_bytes=1409286144 total_nvlink_bytes=1877589632 weight_to_token_ratio=3.009344 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202246141e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32638 payload_rows_remote=28522 dup_rows=130 dup_rows_remote=128 dedup_ratio=0.003967 router_tokens_per_expert=[2111, 2075, 1997, 2038, 2076, 2000, 2045, 2042] payload_rows_per_dst=[3648, 4066, 4040, 3966, 4358, 4062, 4382, 4116] dup_rows_per_dst=[0, 0, 16, 0, 36, 16, 60, 2] combine_rows_per_src=[3648, 4066, 4040, 3966, 4358, 4062, 4382, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2084, 2094, 2057, 2078, 2045, 1972, 2002, 2052] payload_rows_per_dst=[4168, 4188, 4114, 4156, 4090, 3944, 4004, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4168, 4188, 4114, 4156, 4090, 3944, 4004, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28674 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2119, 2047, 2063, 2040, 1981, 2042, 2040, 2052] payload_rows_per_dst=[4238, 4094, 4126, 4080, 3962, 4084, 4080, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4238, 4094, 4126, 4080, 3962, 4084, 4080, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 2046, 1977, 2116, 1983, 2025, 2112, 2064] payload_rows_per_dst=[4122, 4092, 3954, 4232, 3966, 4050, 4224, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 4092, 3954, 4232, 3966, 4050, 4224, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233652224 dispatch_meta_bytes=114600 combine_payload_bytes=233652224 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233766824 ingress_bytes=1642938368 link_bytes=1642938368 token_bytes=467419048 weight_movement_bytes=1409286144 total_nvlink_bytes=1876705192 weight_to_token_ratio=3.015038 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.199769065e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2139, 2037, 2014, 2081, 2018, 2004, 2018, 2073] payload_rows_per_dst=[4278, 4074, 4028, 4162, 4036, 4008, 4036, 4146] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4278, 4074, 4028, 4162, 4036, 4008, 4036, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2073, 2068, 2042, 2024, 2077, 2001, 2044] payload_rows_per_dst=[4110, 4146, 4136, 4084, 4048, 4154, 4002, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4146, 4136, 4084, 4048, 4154, 4002, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32698 payload_rows_remote=28434 dup_rows=70 dup_rows_remote=48 dedup_ratio=0.002136 router_tokens_per_expert=[2058, 2008, 2076, 2023, 2029, 1987, 2048, 2155] payload_rows_per_dst=[4264, 4110, 4050, 3852, 4058, 3974, 4192, 4198] dup_rows_per_dst=[22, 30, 2, 0, 0, 0, 16, 0] combine_rows_per_src=[4264, 4110, 4050, 3852, 4058, 3974, 4192, 4198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235634688 dispatch_meta_bytes=115056 combine_payload_bytes=235634688 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235749744 ingress_bytes=2349563904 link_bytes=2349563904 token_bytes=471384432 weight_movement_bytes=2113929216 total_nvlink_bytes=2585313648 weight_to_token_ratio=4.484512 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315657711e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2063, 2014, 2061, 2005, 2057, 2046, 2094, 2044] payload_rows_per_dst=[4126, 4028, 4122, 4010, 4114, 4092, 4188, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4126, 4028, 4122, 4010, 4114, 4092, 4188, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234897408 dispatch_meta_bytes=114696 combine_payload_bytes=234897408 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235012104 ingress_bytes=3053469696 link_bytes=3053469696 token_bytes=469909512 weight_movement_bytes=2818572288 total_nvlink_bytes=3288481800 weight_to_token_ratio=5.998117 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709815572e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=236044288 dispatch_meta_bytes=115256 combine_payload_bytes=236044288 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=236159544 ingress_bytes=3054616576 link_bytes=3054616576 token_bytes=472203832 weight_movement_bytes=2818572288 total_nvlink_bytes=3290776120 weight_to_token_ratio=5.968974 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710457777e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114424 combine_payload_bytes=234340352 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234454776 ingress_bytes=3052912640 link_bytes=3052912640 token_bytes=468795128 weight_movement_bytes=2818572288 total_nvlink_bytes=3287367416 weight_to_token_ratio=6.012375 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709503644e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234520344 ingress_bytes=3052978176 link_bytes=3052978176 token_bytes=468926232 weight_movement_bytes=2818572288 total_nvlink_bytes=3287498520 weight_to_token_ratio=6.010694 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709540341e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=232931328 dispatch_meta_bytes=113928 combine_payload_bytes=232931328 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233045256 ingress_bytes=2346860544 link_bytes=2346860544 token_bytes=465976584 weight_movement_bytes=2113929216 total_nvlink_bytes=2579905800 weight_to_token_ratio=4.536557 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314143942e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234848184 ingress_bytes=2348662784 link_bytes=2348662784 token_bytes=469581752 weight_movement_bytes=2113929216 total_nvlink_bytes=2583510968 weight_to_token_ratio=4.501728 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315153121e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=28532 dup_rows=76 dup_rows_remote=76 dedup_ratio=0.002319 router_tokens_per_expert=[2004, 2037, 2064, 2084, 2028, 2046, 2049, 2072] payload_rows_per_dst=[3706, 4038, 3816, 4418, 4096, 4366, 4092, 4160] dup_rows_per_dst=[0, 0, 0, 38, 10, 28, 0, 0] combine_rows_per_src=[3706, 4038, 3816, 4418, 4096, 4366, 4092, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28754 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2050, 2059, 2053, 2044, 2049, 2052, 2007, 2070] payload_rows_per_dst=[4100, 4118, 4106, 4088, 4098, 4104, 4014, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4100, 4118, 4106, 4088, 4098, 4104, 4014, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28568 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2100, 2076, 2015, 2072, 2080, 2049, 1958] payload_rows_per_dst=[4068, 4200, 4152, 4030, 4144, 4160, 4098, 3916] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4200, 4152, 4030, 4144, 4160, 4098, 3916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233734144 dispatch_meta_bytes=114432 combine_payload_bytes=233734144 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233848576 ingress_bytes=2347663360 link_bytes=2347663360 token_bytes=467582720 weight_movement_bytes=2113929216 total_nvlink_bytes=2581511936 weight_to_token_ratio=4.520974 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314593485e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2131, 2011, 2090, 2022, 2012, 1994, 2077, 2047] payload_rows_per_dst=[4262, 4022, 4180, 4044, 4024, 3988, 4154, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4262, 4022, 4180, 4044, 4024, 3988, 4154, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2087, 2066, 2095, 2093, 2004, 1968, 2035, 2036] payload_rows_per_dst=[4174, 4132, 4190, 4186, 4008, 3936, 4070, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4174, 4132, 4190, 4186, 4008, 3936, 4070, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28848 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2004, 2061, 2141, 2078, 2055, 1960, 2028, 2057] payload_rows_per_dst=[4008, 4122, 4282, 4156, 4110, 3920, 4056, 4114] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4008, 4122, 4282, 4156, 4110, 3920, 4056, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28798 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2022, 2002, 2111, 2089, 1985, 2126, 2053, 1996] payload_rows_per_dst=[4044, 4004, 4222, 4178, 3970, 4252, 4106, 3992] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4044, 4004, 4222, 4178, 3970, 4252, 4106, 3992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32752 payload_rows_remote=28362 dup_rows=16 dup_rows_remote=10 dedup_ratio=0.000488 router_tokens_per_expert=[2173, 2065, 2002, 1961, 2016, 2066, 2025, 2076] payload_rows_per_dst=[4390, 4130, 4108, 3942, 4018, 4018, 3994, 4152] dup_rows_per_dst=[6, 0, 10, 0, 0, 0, 0, 0] combine_rows_per_src=[4390, 4130, 4108, 3942, 4018, 4018, 3994, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235552768 dispatch_meta_bytes=115016 combine_payload_bytes=235552768 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235667784 ingress_bytes=1644838912 link_bytes=1644838912 token_bytes=471220552 weight_movement_bytes=1409286144 total_nvlink_bytes=1880506696 weight_to_token_ratio=2.990715 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210411318e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234029056 dispatch_meta_bytes=114272 combine_payload_bytes=234029056 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234143328 ingress_bytes=3052601344 link_bytes=3052601344 token_bytes=468172384 weight_movement_bytes=2818572288 total_nvlink_bytes=3286744672 weight_to_token_ratio=6.020373 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709329331e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234192896 dispatch_meta_bytes=114352 combine_payload_bytes=234192896 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234307248 ingress_bytes=4462051328 link_bytes=4462051328 token_bytes=468500144 weight_movement_bytes=4227858432 total_nvlink_bytes=4696358576 weight_to_token_ratio=9.024241 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498562488e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234143744 dispatch_meta_bytes=114328 combine_payload_bytes=234143744 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234258072 ingress_bytes=2348072960 link_bytes=2348072960 token_bytes=468401816 weight_movement_bytes=2113929216 total_nvlink_bytes=2582331032 weight_to_token_ratio=4.513068 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314822844e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=236322816 dispatch_meta_bytes=115392 combine_payload_bytes=236322816 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236438208 ingress_bytes=1645608960 link_bytes=1645608960 token_bytes=472761024 weight_movement_bytes=1409286144 total_nvlink_bytes=1882047168 weight_to_token_ratio=2.980969 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.214723265e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235913216 dispatch_meta_bytes=115192 combine_payload_bytes=235913216 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236028408 ingress_bytes=3054485504 link_bytes=3054485504 token_bytes=471941624 weight_movement_bytes=2818572288 total_nvlink_bytes=3290513912 weight_to_token_ratio=5.972290 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710384382e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=232341504 dispatch_meta_bytes=113488 combine_payload_bytes=232341504 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=232454992 ingress_bytes=5164843008 link_bytes=5164843008 token_bytes=464796496 weight_movement_bytes=4932501504 total_nvlink_bytes=5397298000 weight_to_token_ratio=10.612174 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.892096493e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32640 payload_rows_remote=28432 dup_rows=128 dup_rows_remote=112 dedup_ratio=0.003906 router_tokens_per_expert=[2090, 2005, 1966, 2076, 2120, 2014, 2100, 2013] payload_rows_per_dst=[3550, 3990, 3770, 4360, 4240, 4302, 4220, 4208] dup_rows_per_dst=[0, 22, 2, 32, 0, 52, 4, 16] combine_rows_per_src=[3550, 3990, 3770, 4360, 4240, 4302, 4220, 4208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7522223104 prefetch_bytes=16911433728 reduce_bytes=21139292160 weight_movement_bytes=38050725888 egress_bytes=3762029568 ingress_bytes=41810919424 link_bytes=41810919424 total_nvlink_bytes=45572948992 weight_to_token_ratio=5.058442 prefetch_share_of_ingress=0.404474 dedup_ratio=0.000000 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=30 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.463273100e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2047, 2100, 2024, 2030, 2077, 2018, 2029] payload_rows_per_dst=[4118, 4094, 4200, 4048, 4060, 4154, 4036, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4094, 4200, 4048, 4060, 4154, 4036, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2085, 2023, 2067, 2066, 2028, 2001, 2017, 2097] payload_rows_per_dst=[4170, 4046, 4134, 4132, 4056, 4002, 4034, 4194] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4170, 4046, 4134, 4132, 4056, 4002, 4034, 4194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=232914944 dispatch_meta_bytes=114176 combine_payload_bytes=232914944 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=233029120 ingress_bytes=937558016 link_bytes=937558016 token_bytes=465944064 weight_movement_bytes=704643072 total_nvlink_bytes=1170587136 weight_to_token_ratio=1.512291 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.249933534e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2072, 2080, 2018, 2066, 2070, 2034, 2045, 1999] payload_rows_per_dst=[4144, 4160, 4036, 4132, 4140, 4068, 4090, 3998] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4144, 4160, 4036, 4132, 4140, 4068, 4090, 3998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1999, 2061, 2086, 2062, 1978, 2034, 2074, 2090] payload_rows_per_dst=[3998, 4122, 4172, 4124, 3956, 4068, 4148, 4180] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3998, 4122, 4172, 4124, 3956, 4068, 4148, 4180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32694 payload_rows_remote=28380 dup_rows=74 dup_rows_remote=18 dedup_ratio=0.002258 router_tokens_per_expert=[1966, 2028, 2192, 2019, 2113, 2037, 2025, 2004] payload_rows_per_dst=[4314, 3894, 4236, 4008, 4278, 4070, 4008, 3886] dup_rows_per_dst=[56, 0, 8, 2, 8, 0, 0, 0] combine_rows_per_src=[4314, 3894, 4236, 4008, 4278, 4070, 4008, 3886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28624 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2083, 2098, 1963, 2066, 2072, 2066, 1991] payload_rows_per_dst=[4090, 4166, 4196, 3926, 4132, 4144, 4132, 3982] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4166, 4196, 3926, 4132, 4144, 4132, 3982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235487472 ingress_bytes=2349301760 link_bytes=2349301760 token_bytes=470860016 weight_movement_bytes=2113929216 total_nvlink_bytes=2584789232 weight_to_token_ratio=4.489507 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315510921e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235405512 ingress_bytes=2349219840 link_bytes=2349219840 token_bytes=470696136 weight_movement_bytes=2113929216 total_nvlink_bytes=2584625352 weight_to_token_ratio=4.491070 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315465049e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235487472 ingress_bytes=2349301760 link_bytes=2349301760 token_bytes=470860016 weight_movement_bytes=2113929216 total_nvlink_bytes=2584789232 weight_to_token_ratio=4.489507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315510921e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234766224 ingress_bytes=2348580864 link_bytes=2348580864 token_bytes=469417872 weight_movement_bytes=2113929216 total_nvlink_bytes=2583347088 weight_to_token_ratio=4.503299 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315107249e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232488960 dispatch_meta_bytes=113592 combine_payload_bytes=232488960 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232602552 ingress_bytes=1641775104 link_bytes=1641775104 token_bytes=465091512 weight_movement_bytes=1409286144 total_nvlink_bytes=1874377656 weight_to_token_ratio=3.030127 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.193255272e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=28542 dup_rows=64 dup_rows_remote=62 dedup_ratio=0.001953 router_tokens_per_expert=[2072, 2033, 2117, 2001, 2050, 1938, 2093, 2080] payload_rows_per_dst=[4144, 4064, 4004, 4002, 3918, 4068, 4342, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 36, 26, 2] combine_rows_per_src=[4144, 4064, 4004, 4002, 3918, 4068, 4342, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234487808 dispatch_meta_bytes=114496 combine_payload_bytes=234487808 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234602304 ingress_bytes=2348417024 link_bytes=2348417024 token_bytes=469090112 weight_movement_bytes=2113929216 total_nvlink_bytes=2583019328 weight_to_token_ratio=4.506446 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315015506e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7514291312 prefetch_bytes=19025362944 reduce_bytes=14092861440 weight_movement_bytes=33118224384 egress_bytes=3758062704 ingress_bytes=36874452992 link_bytes=36874452992 total_nvlink_bytes=40632515696 weight_to_token_ratio=4.407365 prefetch_share_of_ingress=0.515950 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=20 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.290509654e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2029, 2036, 2016, 2070, 2071, 2037, 2071] payload_rows_per_dst=[4108, 4058, 4072, 4032, 4140, 4142, 4074, 4142] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4058, 4072, 4032, 4140, 4142, 4074, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7463229128 prefetch_bytes=18320719872 reduce_bytes=19025362944 weight_movement_bytes=37346082816 egress_bytes=3732526792 ingress_bytes=41076785152 link_bytes=41076785152 total_nvlink_bytes=44809311944 weight_to_token_ratio=5.004011 prefetch_share_of_ingress=0.446012 dedup_ratio=0.001980 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=27 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.437580316e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114416 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233930480 ingress_bytes=4461674496 link_bytes=4461674496 token_bytes=467746544 weight_movement_bytes=4227858432 total_nvlink_bytes=4695604976 weight_to_token_ratio=9.038781 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498351478e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28440 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1960, 2164, 1930, 2049, 2126, 2115, 1988, 2052] payload_rows_per_dst=[3920, 4328, 3860, 4098, 4252, 4230, 3976, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3920, 4328, 3860, 4098, 4252, 4230, 3976, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28710 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2000, 2105, 2029, 2066, 2084, 1938, 2084, 2078] payload_rows_per_dst=[4000, 4210, 4058, 4132, 4168, 3876, 4168, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4000, 4210, 4058, 4132, 4168, 3876, 4168, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235176024 ingress_bytes=2348990464 link_bytes=2348990464 token_bytes=470237272 weight_movement_bytes=2113929216 total_nvlink_bytes=2584166488 weight_to_token_ratio=4.495452 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315336608e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7471752256 prefetch_bytes=18320719872 reduce_bytes=15502147584 weight_movement_bytes=33822867456 egress_bytes=3736790080 ingress_bytes=37557829632 link_bytes=37557829632 total_nvlink_bytes=41294619712 weight_to_token_ratio=4.526765 prefetch_share_of_ingress=0.487800 dedup_ratio=0.002537 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=22 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.314426053e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28742 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2092, 2044, 2068, 2096, 2039, 2013, 1960, 2072] payload_rows_per_dst=[4184, 4088, 4136, 4192, 4078, 4026, 3920, 4144] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4184, 4088, 4136, 4192, 4078, 4026, 3920, 4144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232980480 dispatch_meta_bytes=113760 combine_payload_bytes=232980480 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233094240 ingress_bytes=2346909696 link_bytes=2346909696 token_bytes=466074720 weight_movement_bytes=2113929216 total_nvlink_bytes=2580003936 weight_to_token_ratio=4.535602 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314171465e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235192320 dispatch_meta_bytes=114840 combine_payload_bytes=235192320 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235307160 ingress_bytes=3053764608 link_bytes=3053764608 token_bytes=470499480 weight_movement_bytes=2818572288 total_nvlink_bytes=3289071768 weight_to_token_ratio=5.990596 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709980710e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7519928784 prefetch_bytes=17616076800 reduce_bytes=14797504512 weight_movement_bytes=32413581312 egress_bytes=3760882128 ingress_bytes=36172627968 link_bytes=36172627968 total_nvlink_bytes=39933510096 weight_to_token_ratio=4.310357 prefetch_share_of_ingress=0.487000 dedup_ratio=0.000000 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=21 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.265947609e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7520387648 prefetch_bytes=19025362944 reduce_bytes=14797504512 weight_movement_bytes=33822867456 egress_bytes=3761111616 ingress_bytes=37582143488 link_bytes=37582143488 total_nvlink_bytes=41343255104 weight_to_token_ratio=4.497490 prefetch_share_of_ingress=0.506234 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=21 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.315276975e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235454464 dispatch_meta_bytes=114968 combine_payload_bytes=235454464 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235569432 ingress_bytes=2349383680 link_bytes=2349383680 token_bytes=471023896 weight_movement_bytes=2113929216 total_nvlink_bytes=2584953112 weight_to_token_ratio=4.487945 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315556793e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7523468592 prefetch_bytes=20434649088 reduce_bytes=21843935232 weight_movement_bytes=42278584320 egress_bytes=3762652464 ingress_bytes=46039400448 link_bytes=46039400448 total_nvlink_bytes=49802052912 weight_to_token_ratio=5.619560 prefetch_share_of_ingress=0.443851 dedup_ratio=0.000000 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=31 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.611258904e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7519797680 prefetch_bytes=18320719872 reduce_bytes=26776436736 weight_movement_bytes=45097156608 egress_bytes=3760816560 ingress_bytes=48856137728 link_bytes=48856137728 total_nvlink_bytes=52616954288 weight_to_token_ratio=5.997124 prefetch_share_of_ingress=0.374993 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=38 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.709837361e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none - [2026-09-16 21:35:54] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 571.4 | throughput per GPU (TFLOP/s/GPU): 583.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.836405E+00 | loss scale: 1.0 | grad norm: 4.041 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32698 payload_rows_remote=28464 dup_rows=70 dup_rows_remote=40 dedup_ratio=0.002136 router_tokens_per_expert=[2172, 2035, 1995, 2011, 2021, 2042, 2092, 2016] payload_rows_per_dst=[4108, 3864, 3774, 4022, 4240, 4072, 4384, 4234] dup_rows_per_dst=[0, 0, 0, 0, 18, 0, 22, 30] combine_rows_per_src=[4108, 3864, 3774, 4022, 4240, 4072, 4384, 4234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2084, 2042, 2015, 2054, 2000, 2063, 2083, 2043] payload_rows_per_dst=[4168, 4084, 4030, 4108, 4000, 4126, 4166, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4168, 4084, 4030, 4108, 4000, 4126, 4166, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28602 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2033, 2040, 2000, 2083, 2092, 2060, 2049, 2027] payload_rows_per_dst=[4066, 4080, 4000, 4166, 4184, 4120, 4098, 4054] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4066, 4080, 4000, 4166, 4184, 4120, 4098, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2019, 2037, 2109, 2087, 1973, 2088, 2018, 2053] payload_rows_per_dst=[4038, 4074, 4218, 4174, 3946, 4176, 4036, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4038, 4074, 4218, 4174, 3946, 4176, 4036, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2020, 2054, 2106, 2055, 2043, 2092, 1993, 2021] payload_rows_per_dst=[4040, 4108, 4212, 4110, 4086, 4184, 3986, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4040, 4108, 4212, 4110, 4086, 4184, 3986, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32668 payload_rows_remote=28456 dup_rows=100 dup_rows_remote=92 dedup_ratio=0.003052 router_tokens_per_expert=[2080, 2058, 2006, 2047, 2061, 2010, 2085, 2037] payload_rows_per_dst=[4212, 4400, 4096, 3988, 4004, 3918, 4044, 4006] dup_rows_per_dst=[8, 54, 22, 16, 0, 0, 0, 0] combine_rows_per_src=[4212, 4400, 4096, 3988, 4004, 3918, 4044, 4006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2013, 2085, 2062, 2049, 2049, 2010, 2045] payload_rows_per_dst=[4142, 4026, 4170, 4124, 4098, 4098, 4020, 4090] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4026, 4170, 4124, 4098, 4098, 4020, 4090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233177088 dispatch_meta_bytes=114016 combine_payload_bytes=233177088 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233291104 ingress_bytes=2347106304 link_bytes=2347106304 token_bytes=466468192 weight_movement_bytes=2113929216 total_nvlink_bytes=2580397408 weight_to_token_ratio=4.531776 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314281557e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114568 combine_payload_bytes=234635264 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234749832 ingress_bytes=3053207552 link_bytes=3053207552 token_bytes=469385096 weight_movement_bytes=2818572288 total_nvlink_bytes=3287957384 weight_to_token_ratio=6.004818 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709668782e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234307584 dispatch_meta_bytes=114408 combine_payload_bytes=234307584 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234421992 ingress_bytes=2348236800 link_bytes=2348236800 token_bytes=468729576 weight_movement_bytes=2113929216 total_nvlink_bytes=2582658792 weight_to_token_ratio=4.509912 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314914588e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235487472 ingress_bytes=3053944832 link_bytes=3053944832 token_bytes=470860016 weight_movement_bytes=2818572288 total_nvlink_bytes=3289432304 weight_to_token_ratio=5.986009 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710081628e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114640 combine_payload_bytes=234782720 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234897360 ingress_bytes=2348711936 link_bytes=2348711936 token_bytes=469680080 weight_movement_bytes=2113929216 total_nvlink_bytes=2583609296 weight_to_token_ratio=4.500785 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315180644e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233111552 dispatch_meta_bytes=114192 combine_payload_bytes=233111552 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233225744 ingress_bytes=3051683840 link_bytes=3051683840 token_bytes=466337296 weight_movement_bytes=2818572288 total_nvlink_bytes=3284909584 weight_to_token_ratio=6.044064 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708815567e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234979320 ingress_bytes=2348793856 link_bytes=2348793856 token_bytes=469843960 weight_movement_bytes=2113929216 total_nvlink_bytes=2583773176 weight_to_token_ratio=4.499215 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315226516e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28522 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1993, 2039, 2123, 2030, 2096, 2037, 2006, 2060] payload_rows_per_dst=[3986, 4078, 4246, 4060, 4192, 4074, 4012, 4120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3986, 4078, 4246, 4060, 4192, 4074, 4012, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32732 payload_rows_remote=28600 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.001099 router_tokens_per_expert=[2061, 2023, 2024, 2043, 1978, 2059, 2130, 2066] payload_rows_per_dst=[3810, 4046, 4042, 4090, 4078, 4140, 4394, 4132] dup_rows_per_dst=[0, 0, 0, 0, 22, 2, 12, 0] combine_rows_per_src=[3810, 4046, 4042, 4090, 4078, 4140, 4394, 4132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28528 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2022, 2026, 2010, 2051, 2068, 2120, 2020, 2067] payload_rows_per_dst=[4044, 4052, 4020, 4102, 4136, 4240, 4040, 4134] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4044, 4052, 4020, 4102, 4136, 4240, 4040, 4134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28668 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2040, 2081, 2042, 2013, 2057, 2050, 2050] payload_rows_per_dst=[4102, 4080, 4162, 4084, 4026, 4114, 4100, 4100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4080, 4162, 4084, 4026, 4114, 4100, 4100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32690 payload_rows_remote=28420 dup_rows=78 dup_rows_remote=72 dedup_ratio=0.002380 router_tokens_per_expert=[2104, 2045, 2086, 2078, 2024, 1995, 2008, 2044] payload_rows_per_dst=[4270, 4216, 4338, 4222, 3922, 3836, 3858, 4028] dup_rows_per_dst=[6, 28, 32, 12, 0, 0, 0, 0] combine_rows_per_src=[4270, 4216, 4338, 4222, 3922, 3836, 3858, 4028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28542 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2083, 2042, 2005, 2113, 2101, 2013, 2048, 1979] payload_rows_per_dst=[4166, 4084, 4010, 4226, 4202, 4026, 4096, 3958] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4166, 4084, 4010, 4226, 4202, 4026, 4096, 3958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233652224 dispatch_meta_bytes=114088 combine_payload_bytes=233652224 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233766312 ingress_bytes=3052224512 link_bytes=3052224512 token_bytes=467418536 weight_movement_bytes=2818572288 total_nvlink_bytes=3285990824 weight_to_token_ratio=6.030082 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709118321e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2093, 2015, 1977, 2094, 2070, 2023, 2091] payload_rows_per_dst=[4042, 4186, 4030, 3954, 4188, 4140, 4046, 4182] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4186, 4030, 3954, 4188, 4140, 4046, 4182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28744 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2094, 2012, 2005, 2022, 2028, 2074, 2053, 2096] payload_rows_per_dst=[4188, 4024, 4010, 4044, 4056, 4148, 4106, 4192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4188, 4024, 4010, 4044, 4056, 4148, 4106, 4192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234291200 dispatch_meta_bytes=114544 combine_payload_bytes=234291200 prefetch_bytes=0 reduce_bytes=2113929216 egress_bytes=234405744 ingress_bytes=2348220416 link_bytes=2348220416 token_bytes=468696944 weight_movement_bytes=2113929216 total_nvlink_bytes=2582626160 weight_to_token_ratio=4.510226 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314905413e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233701376 dispatch_meta_bytes=114112 combine_payload_bytes=233701376 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233815488 ingress_bytes=3052273664 link_bytes=3052273664 token_bytes=467516864 weight_movement_bytes=2818572288 total_nvlink_bytes=3286089152 weight_to_token_ratio=6.028814 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709145844e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234848256 dispatch_meta_bytes=114672 combine_payload_bytes=234848256 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234962928 ingress_bytes=2348777472 link_bytes=2348777472 token_bytes=469811184 weight_movement_bytes=2113929216 total_nvlink_bytes=2583740400 weight_to_token_ratio=4.499529 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315217342e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=232816640 dispatch_meta_bytes=113968 combine_payload_bytes=232816640 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=232930608 ingress_bytes=4460675072 link_bytes=4460675072 token_bytes=465747248 weight_movement_bytes=4227858432 total_nvlink_bytes=4693605680 weight_to_token_ratio=9.077581 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.497791843e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114168 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233930232 ingress_bytes=1643102208 link_bytes=1643102208 token_bytes=467746296 weight_movement_bytes=1409286144 total_nvlink_bytes=1877032440 weight_to_token_ratio=3.012928 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.200686500e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234127360 dispatch_meta_bytes=114320 combine_payload_bytes=234127360 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234241680 ingress_bytes=2348056576 link_bytes=2348056576 token_bytes=468369040 weight_movement_bytes=2113929216 total_nvlink_bytes=2582298256 weight_to_token_ratio=4.513384 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314813670e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235470848 dispatch_meta_bytes=114976 combine_payload_bytes=235470848 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235585824 ingress_bytes=2349400064 link_bytes=2349400064 token_bytes=471056672 weight_movement_bytes=2113929216 total_nvlink_bytes=2584985888 weight_to_token_ratio=4.487632 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315565967e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2070, 2026, 2062, 2017, 2069, 2061, 2058, 2021] payload_rows_per_dst=[4140, 4052, 4124, 4034, 4138, 4122, 4116, 4042] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4052, 4124, 4034, 4138, 4122, 4116, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28500 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2067, 2068, 1973, 2038, 1999, 2134, 2048] payload_rows_per_dst=[4114, 4134, 4136, 3946, 4076, 3998, 4268, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4134, 4136, 3946, 4076, 3998, 4268, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28542 dup_rows=62 dup_rows_remote=36 dedup_ratio=0.001892 router_tokens_per_expert=[2037, 2063, 2031, 2046, 2015, 1994, 2078, 2120] payload_rows_per_dst=[4164, 4198, 4062, 4058, 3948, 3988, 4196, 4092] dup_rows_per_dst=[26, 6, 0, 0, 0, 0, 30, 0] combine_rows_per_src=[4164, 4198, 4062, 4058, 3948, 3988, 4196, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32620 payload_rows_remote=28486 dup_rows=148 dup_rows_remote=144 dedup_ratio=0.004517 router_tokens_per_expert=[2058, 2079, 2063, 2097, 2029, 2033, 1977, 2048] payload_rows_per_dst=[3926, 3610, 4200, 4316, 4126, 4342, 3966, 4134] dup_rows_per_dst=[0, 0, 46, 30, 16, 48, 4, 4] combine_rows_per_src=[3926, 3610, 4200, 4316, 4126, 4342, 3966, 4134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28686 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2094, 2084, 2050, 2060, 2017, 2041, 2018, 2020] payload_rows_per_dst=[4188, 4168, 4100, 4120, 4034, 4082, 4036, 4040] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4188, 4168, 4100, 4120, 4034, 4082, 4036, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28600 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 2118, 2062, 1972, 2084, 2041, 1997, 2063] payload_rows_per_dst=[4094, 4236, 4124, 3944, 4168, 4082, 3994, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 4236, 4124, 3944, 4168, 4082, 3994, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234766224 ingress_bytes=3053223936 link_bytes=3053223936 token_bytes=469417872 weight_movement_bytes=2818572288 total_nvlink_bytes=3287990160 weight_to_token_ratio=6.004399 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709677956e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28618 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2040, 2075, 1952, 2104, 2102, 2038, 2008, 2065] payload_rows_per_dst=[4080, 4150, 3904, 4208, 4204, 4076, 4016, 4130] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4080, 4150, 3904, 4208, 4204, 4076, 4016, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114000 combine_payload_bytes=233472000 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233586000 ingress_bytes=2347401216 link_bytes=2347401216 token_bytes=467058000 weight_movement_bytes=2113929216 total_nvlink_bytes=2580987216 weight_to_token_ratio=4.526053 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314446696e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2028, 2084, 2054, 2006, 2048, 2059, 2030, 2075] payload_rows_per_dst=[4056, 4168, 4108, 4012, 4096, 4118, 4060, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4056, 4168, 4108, 4012, 4096, 4118, 4060, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114312 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233930376 ingress_bytes=2347745280 link_bytes=2347745280 token_bytes=467746440 weight_movement_bytes=2113929216 total_nvlink_bytes=2581675656 weight_to_token_ratio=4.519391 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314639357e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233357312 dispatch_meta_bytes=114520 combine_payload_bytes=233357312 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=233471832 ingress_bytes=1642643456 link_bytes=1642643456 token_bytes=466829144 weight_movement_bytes=1409286144 total_nvlink_bytes=1876115288 weight_to_token_ratio=3.018848 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.198117680e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234995712 dispatch_meta_bytes=114744 combine_payload_bytes=234995712 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235110456 ingress_bytes=1644281856 link_bytes=1644281856 token_bytes=470106168 weight_movement_bytes=1409286144 total_nvlink_bytes=1879392312 weight_to_token_ratio=2.997804 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207292037e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234291200 dispatch_meta_bytes=114400 combine_payload_bytes=234291200 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234405600 ingress_bytes=2348220416 link_bytes=2348220416 token_bytes=468696800 weight_movement_bytes=2113929216 total_nvlink_bytes=2582626016 weight_to_token_ratio=4.510228 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314905413e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234438656 dispatch_meta_bytes=114472 combine_payload_bytes=234438656 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234553128 ingress_bytes=4462297088 link_bytes=4462297088 token_bytes=468991784 weight_movement_bytes=4227858432 total_nvlink_bytes=4696850216 weight_to_token_ratio=9.014781 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498700104e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235684176 ingress_bytes=3054141440 link_bytes=3054141440 token_bytes=471253328 weight_movement_bytes=2818572288 total_nvlink_bytes=3289825616 weight_to_token_ratio=5.981013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710191720e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 2049, 2044, 2067, 2050, 2017, 2099, 1998] payload_rows_per_dst=[4120, 4098, 4088, 4134, 4100, 4034, 4198, 3996] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 4098, 4088, 4134, 4100, 4034, 4198, 3996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32700 payload_rows_remote=28630 dup_rows=68 dup_rows_remote=66 dedup_ratio=0.002075 router_tokens_per_expert=[2029, 2035, 2044, 2119, 2074, 2013, 2026, 2044] payload_rows_per_dst=[4070, 4248, 4210, 4244, 3884, 4026, 4052, 3966] dup_rows_per_dst=[2, 34, 30, 2, 0, 0, 0, 0] combine_rows_per_src=[4070, 4248, 4210, 4244, 3884, 4026, 4052, 3966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2081, 2021, 2052, 2070, 2006, 2064, 2020, 2070] payload_rows_per_dst=[4162, 4042, 4104, 4140, 4012, 4128, 4040, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4162, 4042, 4104, 4140, 4012, 4128, 4040, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28746 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2062, 2068, 2002, 2102, 2032, 2043, 2011, 2064] payload_rows_per_dst=[4124, 4136, 4004, 4204, 4064, 4086, 4022, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4124, 4136, 4004, 4204, 4064, 4086, 4022, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32612 payload_rows_remote=28368 dup_rows=156 dup_rows_remote=126 dedup_ratio=0.004761 router_tokens_per_expert=[2071, 2042, 2046, 2015, 2105, 2027, 2028, 2050] payload_rows_per_dst=[3662, 4084, 4156, 3688, 4156, 4234, 4388, 4244] dup_rows_per_dst=[0, 0, 16, 0, 4, 38, 68, 30] combine_rows_per_src=[3662, 4084, 4156, 3688, 4156, 4234, 4388, 4244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235061280 ingress_bytes=2348875776 link_bytes=2348875776 token_bytes=470007840 weight_movement_bytes=2113929216 total_nvlink_bytes=2583937056 weight_to_token_ratio=4.497647 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315272388e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28770 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2114, 2015, 2069, 2028, 2061, 1999, 2023, 2075] payload_rows_per_dst=[4228, 4030, 4138, 4056, 4122, 3998, 4046, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4228, 4030, 4138, 4056, 4122, 3998, 4046, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28618 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2063, 2078, 2014, 2075, 2142, 2017, 2002, 1993] payload_rows_per_dst=[4126, 4156, 4028, 4150, 4284, 4034, 4004, 3986] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4126, 4156, 4028, 4150, 4284, 4034, 4004, 3986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28820 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2118, 1974, 2028, 2023, 2036, 2066, 2031, 2108] payload_rows_per_dst=[4236, 3948, 4056, 4046, 4072, 4132, 4062, 4216] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4236, 3948, 4056, 4046, 4072, 4132, 4062, 4216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114784 combine_payload_bytes=234536960 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234651744 ingress_bytes=4462395392 link_bytes=4462395392 token_bytes=469188704 weight_movement_bytes=4227858432 total_nvlink_bytes=4697047136 weight_to_token_ratio=9.010998 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498755150e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=235684176 ingress_bytes=4463427584 link_bytes=4463427584 token_bytes=471253328 weight_movement_bytes=4227858432 total_nvlink_bytes=4699111760 weight_to_token_ratio=8.971520 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499333134e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235487232 dispatch_meta_bytes=114984 combine_payload_bytes=235487232 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235602216 ingress_bytes=1644773376 link_bytes=1644773376 token_bytes=471089448 weight_movement_bytes=1409286144 total_nvlink_bytes=1880375592 weight_to_token_ratio=2.991547 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210044343e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=232390656 dispatch_meta_bytes=113976 combine_payload_bytes=232390656 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232504632 ingress_bytes=2346319872 link_bytes=2346319872 token_bytes=464895288 weight_movement_bytes=2113929216 total_nvlink_bytes=2578824504 weight_to_token_ratio=4.547108 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313841188e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235683840 dispatch_meta_bytes=115080 combine_payload_bytes=235683840 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235798920 ingress_bytes=1644969984 link_bytes=1644969984 token_bytes=471482760 weight_movement_bytes=1409286144 total_nvlink_bytes=1880768904 weight_to_token_ratio=2.989051 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211145266e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234438656 dispatch_meta_bytes=114472 combine_payload_bytes=234438656 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234553128 ingress_bytes=3757654016 link_bytes=3757654016 token_bytes=468991784 weight_movement_bytes=3523215360 total_nvlink_bytes=3992207144 weight_to_token_ratio=7.512318 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104129397e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=236093440 dispatch_meta_bytes=115280 combine_payload_bytes=236093440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236208720 ingress_bytes=1645379584 link_bytes=1645379584 token_bytes=472302160 weight_movement_bytes=1409286144 total_nvlink_bytes=1881588304 weight_to_token_ratio=2.983866 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213438855e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28742 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2079, 2045, 2013, 2119, 2085, 2046, 2043, 1954] payload_rows_per_dst=[4158, 4090, 4026, 4238, 4170, 4092, 4086, 3908] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4158, 4090, 4026, 4238, 4170, 4092, 4086, 3908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2028, 2103, 1980, 2034, 2074, 2053, 2054] payload_rows_per_dst=[4116, 4056, 4206, 3960, 4068, 4148, 4106, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4056, 4206, 3960, 4068, 4148, 4106, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32744 payload_rows_remote=28532 dup_rows=24 dup_rows_remote=12 dedup_ratio=0.000732 router_tokens_per_expert=[2076, 1971, 2080, 2100, 2007, 2073, 1997, 2080] payload_rows_per_dst=[4212, 4050, 4186, 4098, 4014, 4086, 4002, 4096] dup_rows_per_dst=[12, 12, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4212, 4050, 4186, 4098, 4014, 4086, 4002, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1989, 2017, 2037, 2178, 2055, 2028, 2082, 1998] payload_rows_per_dst=[3978, 4034, 4074, 4356, 4110, 4056, 4164, 3996] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3978, 4034, 4074, 4356, 4110, 4056, 4164, 3996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=28500 dup_rows=64 dup_rows_remote=40 dedup_ratio=0.001953 router_tokens_per_expert=[2064, 2008, 2021, 2038, 2093, 2079, 2052, 2029] payload_rows_per_dst=[3870, 4152, 4016, 3850, 4296, 4176, 4140, 4204] dup_rows_per_dst=[0, 14, 0, 0, 24, 2, 0, 24] combine_rows_per_src=[3870, 4152, 4016, 3850, 4296, 4176, 4140, 4204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28720 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2087, 2015, 2049, 2024, 2039, 2080, 2061, 2029] payload_rows_per_dst=[4174, 4030, 4098, 4048, 4078, 4160, 4122, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4174, 4030, 4098, 4048, 4078, 4160, 4122, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235454464 dispatch_meta_bytes=114968 combine_payload_bytes=235454464 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235569432 ingress_bytes=1644740608 link_bytes=1644740608 token_bytes=471023896 weight_movement_bytes=1409286144 total_nvlink_bytes=1880310040 weight_to_token_ratio=2.991963 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209860856e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233734144 dispatch_meta_bytes=114176 combine_payload_bytes=233734144 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233848320 ingress_bytes=3052306432 link_bytes=3052306432 token_bytes=467582464 weight_movement_bytes=2818572288 total_nvlink_bytes=3286154752 weight_to_token_ratio=6.027968 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709164192e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28598 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2053, 2085, 1991, 2154, 2012, 2020, 2037, 2032] payload_rows_per_dst=[4106, 4170, 3982, 4308, 4024, 4040, 4074, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4106, 4170, 3982, 4308, 4024, 4040, 4074, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235225200 ingress_bytes=1644396544 link_bytes=1644396544 token_bytes=470335600 weight_movement_bytes=1409286144 total_nvlink_bytes=1879621744 weight_to_token_ratio=2.996342 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207934242e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234438384 ingress_bytes=1643610112 link_bytes=1643610112 token_bytes=468762352 weight_movement_bytes=1409286144 total_nvlink_bytes=1878048496 weight_to_token_ratio=3.006398 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203530551e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2103, 2053, 2063, 1984, 2040, 2044, 2029, 2068] payload_rows_per_dst=[4206, 4106, 4126, 3968, 4080, 4088, 4058, 4136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4206, 4106, 4126, 3968, 4080, 4088, 4058, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114160 combine_payload_bytes=233472000 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233586160 ingress_bytes=3052044288 link_bytes=3052044288 token_bytes=467058160 weight_movement_bytes=2818572288 total_nvlink_bytes=3285630448 weight_to_token_ratio=6.034735 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709017403e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235274240 dispatch_meta_bytes=114880 combine_payload_bytes=235274240 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=235389120 ingress_bytes=4463132672 link_bytes=4463132672 token_bytes=470663360 weight_movement_bytes=4227858432 total_nvlink_bytes=4698521792 weight_to_token_ratio=8.982765 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499167996e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114392 combine_payload_bytes=234274816 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234389208 ingress_bytes=1643560960 link_bytes=1643560960 token_bytes=468664024 weight_movement_bytes=1409286144 total_nvlink_bytes=1877950168 weight_to_token_ratio=3.007029 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203255320e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235061280 ingress_bytes=3053518848 link_bytes=3053518848 token_bytes=470007840 weight_movement_bytes=2818572288 total_nvlink_bytes=3288580128 weight_to_token_ratio=5.996862 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709843095e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32656 payload_rows_remote=28584 dup_rows=112 dup_rows_remote=104 dedup_ratio=0.003418 router_tokens_per_expert=[1990, 2085, 2043, 2015, 2102, 2074, 2034, 2041] payload_rows_per_dst=[4072, 4288, 4182, 4210, 4082, 3786, 4020, 4016] dup_rows_per_dst=[8, 22, 42, 38, 0, 0, 2, 0] combine_rows_per_src=[4072, 4288, 4182, 4210, 4082, 3786, 4020, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 2072, 2040, 2090, 2037, 2006, 2051, 2041] payload_rows_per_dst=[4094, 4144, 4080, 4180, 4074, 4012, 4102, 4082] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 4144, 4080, 4180, 4074, 4012, 4102, 4082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2109, 2055, 1970, 2065, 2009, 2089, 1996] payload_rows_per_dst=[4182, 4218, 4110, 3940, 4130, 4018, 4178, 3992] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4218, 4110, 3940, 4130, 4018, 4178, 3992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28612 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2077, 2000, 1969, 2021, 2100, 2055, 2078, 2084] payload_rows_per_dst=[4154, 4000, 3938, 4042, 4200, 4110, 4156, 4168] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4154, 4000, 3938, 4042, 4200, 4110, 4156, 4168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32738 payload_rows_remote=28408 dup_rows=30 dup_rows_remote=18 dedup_ratio=0.000916 router_tokens_per_expert=[1993, 1960, 2083, 2075, 1963, 2147, 2041, 2122] payload_rows_per_dst=[3986, 3832, 4166, 4150, 3824, 4366, 4084, 4330] dup_rows_per_dst=[0, 0, 0, 0, 0, 16, 2, 12] combine_rows_per_src=[3986, 3832, 4166, 4150, 3824, 4366, 4084, 4330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28590 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2079, 2084, 2089, 2095, 1981, 1978, 2037] payload_rows_per_dst=[4082, 4158, 4168, 4178, 4190, 3962, 3956, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4158, 4168, 4178, 4190, 3962, 3956, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234160128 dispatch_meta_bytes=114752 combine_payload_bytes=234160128 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234274880 ingress_bytes=1643446272 link_bytes=1643446272 token_bytes=468435008 weight_movement_bytes=1409286144 total_nvlink_bytes=1877721152 weight_to_token_ratio=3.008499 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202613115e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235126848 ingress_bytes=1644298240 link_bytes=1644298240 token_bytes=470138944 weight_movement_bytes=1409286144 total_nvlink_bytes=1879425088 weight_to_token_ratio=2.997595 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207383780e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234717048 ingress_bytes=3053174784 link_bytes=3053174784 token_bytes=469319544 weight_movement_bytes=2818572288 total_nvlink_bytes=3287891832 weight_to_token_ratio=6.005657 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709650433e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2019, 2061, 1950, 2079, 2083, 2045, 2088, 2059] payload_rows_per_dst=[4038, 4122, 3900, 4158, 4166, 4090, 4176, 4118] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4038, 4122, 3900, 4158, 4166, 4090, 4176, 4118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234389504 dispatch_meta_bytes=114448 combine_payload_bytes=234389504 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234503952 ingress_bytes=3052961792 link_bytes=3052961792 token_bytes=468893456 weight_movement_bytes=2818572288 total_nvlink_bytes=3287465744 weight_to_token_ratio=6.011115 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709531167e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28492 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2088, 2025, 2026, 2012, 2029, 2138, 2038, 2028] payload_rows_per_dst=[4176, 4050, 4052, 4024, 4058, 4276, 4076, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4176, 4050, 4052, 4024, 4058, 4276, 4076, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=232718336 dispatch_meta_bytes=113704 combine_payload_bytes=232718336 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=232832040 ingress_bytes=3051290624 link_bytes=3051290624 token_bytes=465550376 weight_movement_bytes=2818572288 total_nvlink_bytes=3284122664 weight_to_token_ratio=6.054280 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708595382e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234209280 dispatch_meta_bytes=114360 combine_payload_bytes=234209280 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234323640 ingress_bytes=1643495424 link_bytes=1643495424 token_bytes=468532920 weight_movement_bytes=1409286144 total_nvlink_bytes=1877819064 weight_to_token_ratio=3.007870 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202888346e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32656 payload_rows_remote=28516 dup_rows=112 dup_rows_remote=110 dedup_ratio=0.003418 router_tokens_per_expert=[2070, 2047, 1996, 2086, 2038, 2054, 2059, 2034] payload_rows_per_dst=[4140, 4546, 4212, 4060, 3830, 3826, 3994, 4048] dup_rows_per_dst=[2, 68, 42, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4546, 4212, 4060, 3830, 3826, 3994, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234668032 dispatch_meta_bytes=114584 combine_payload_bytes=234668032 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234782616 ingress_bytes=2348597248 link_bytes=2348597248 token_bytes=469450648 weight_movement_bytes=2113929216 total_nvlink_bytes=2583379864 weight_to_token_ratio=4.502985 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315116424e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233406464 dispatch_meta_bytes=113968 combine_payload_bytes=233406464 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233520432 ingress_bytes=3756621824 link_bytes=3756621824 token_bytes=466926896 weight_movement_bytes=3523215360 total_nvlink_bytes=3990142256 weight_to_token_ratio=7.545540 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103551412e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28590 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2054, 2058, 1999, 2089, 2066, 2064, 1999] payload_rows_per_dst=[4110, 4108, 4116, 3998, 4178, 4132, 4128, 3998] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4108, 4116, 3998, 4178, 4132, 4128, 3998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28558 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 2039, 2105, 2014, 2059, 2072, 2059, 2001] payload_rows_per_dst=[4070, 4078, 4210, 4028, 4118, 4144, 4118, 4002] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 4078, 4210, 4028, 4118, 4144, 4118, 4002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233603072 dispatch_meta_bytes=114504 combine_payload_bytes=233603072 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233717576 ingress_bytes=2347532288 link_bytes=2347532288 token_bytes=467320648 weight_movement_bytes=2113929216 total_nvlink_bytes=2581249864 weight_to_token_ratio=4.523509 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314520090e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28794 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 1960, 2009, 2116, 2156, 2086, 1987, 2032] payload_rows_per_dst=[4076, 3920, 4018, 4232, 4312, 4172, 3974, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 3920, 4018, 4232, 4312, 4172, 3974, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2069, 2035, 2034, 2026, 2052, 2066, 2057] payload_rows_per_dst=[4090, 4138, 4070, 4068, 4052, 4104, 4132, 4114] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4138, 4070, 4068, 4052, 4104, 4132, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32742 payload_rows_remote=28552 dup_rows=26 dup_rows_remote=12 dedup_ratio=0.000793 router_tokens_per_expert=[2014, 2019, 2069, 2112, 2005, 2079, 2055, 2031] payload_rows_per_dst=[3958, 4038, 4086, 4084, 4056, 4158, 4172, 4190] dup_rows_per_dst=[0, 0, 0, 0, 6, 0, 6, 14] combine_rows_per_src=[3958, 4038, 4086, 4084, 4056, 4158, 4172, 4190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234209280 dispatch_meta_bytes=114360 combine_payload_bytes=234209280 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234323640 ingress_bytes=2348138496 link_bytes=2348138496 token_bytes=468532920 weight_movement_bytes=2113929216 total_nvlink_bytes=2582462136 weight_to_token_ratio=4.511805 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314859542e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233947136 dispatch_meta_bytes=114232 combine_payload_bytes=233947136 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234061368 ingress_bytes=3052519424 link_bytes=3052519424 token_bytes=468008504 weight_movement_bytes=2818572288 total_nvlink_bytes=3286580792 weight_to_token_ratio=6.022481 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709283459e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235880448 dispatch_meta_bytes=115176 combine_payload_bytes=235880448 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235995624 ingress_bytes=2349809664 link_bytes=2349809664 token_bytes=471876072 weight_movement_bytes=2113929216 total_nvlink_bytes=2585805288 weight_to_token_ratio=4.479840 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315795326e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28666 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2023, 2051, 2079, 2088, 1972, 2092, 2059, 2020] payload_rows_per_dst=[4046, 4102, 4158, 4176, 3944, 4184, 4118, 4040] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4046, 4102, 4158, 4176, 3944, 4184, 4118, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2022, 2014, 2023, 2096, 2020, 2092, 2037] payload_rows_per_dst=[4160, 4044, 4028, 4046, 4192, 4040, 4184, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4044, 4028, 4046, 4192, 4040, 4184, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235225200 ingress_bytes=3053682688 link_bytes=3053682688 token_bytes=470335600 weight_movement_bytes=2818572288 total_nvlink_bytes=3288907888 weight_to_token_ratio=5.992683 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709934838e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233897984 dispatch_meta_bytes=114256 combine_payload_bytes=233897984 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234012240 ingress_bytes=3052470272 link_bytes=3052470272 token_bytes=467910224 weight_movement_bytes=2818572288 total_nvlink_bytes=3286482512 weight_to_token_ratio=6.023746 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709255936e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28424 dup_rows=62 dup_rows_remote=44 dedup_ratio=0.001892 router_tokens_per_expert=[2061, 2011, 2031, 2021, 2045, 2087, 2088, 2040] payload_rows_per_dst=[4282, 4270, 3882, 3940, 4152, 4190, 4088, 3902] dup_rows_per_dst=[18, 42, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[4282, 4270, 3882, 3940, 4152, 4190, 4088, 3902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234831872 dispatch_meta_bytes=114664 combine_payload_bytes=234831872 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234946536 ingress_bytes=1644118016 link_bytes=1644118016 token_bytes=469778408 weight_movement_bytes=1409286144 total_nvlink_bytes=1879064552 weight_to_token_ratio=2.999896 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206374601e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2100, 2036, 2073, 2028, 1997, 2080, 2046] payload_rows_per_dst=[4048, 4200, 4072, 4146, 4056, 3994, 4160, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4200, 4072, 4146, 4056, 3994, 4160, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235454688 ingress_bytes=3053912064 link_bytes=3053912064 token_bytes=470794464 weight_movement_bytes=2818572288 total_nvlink_bytes=3289366752 weight_to_token_ratio=5.986842 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710063279e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2093, 1902, 2019, 2052, 2099, 2076, 2030, 2113] payload_rows_per_dst=[4186, 3804, 4038, 4104, 4198, 4152, 4060, 4226] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4186, 3804, 4038, 4104, 4198, 4152, 4060, 4226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=232849408 dispatch_meta_bytes=113872 combine_payload_bytes=232849408 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=232963280 ingress_bytes=3051421696 link_bytes=3051421696 token_bytes=465812688 weight_movement_bytes=2818572288 total_nvlink_bytes=3284384976 weight_to_token_ratio=6.050871 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708668777e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28686 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2027, 2109, 2042, 2021, 2065, 2041, 2058] payload_rows_per_dst=[4042, 4054, 4218, 4084, 4042, 4130, 4082, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4054, 4218, 4084, 4042, 4130, 4082, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1946, 2059, 2096, 2061, 2057, 2073, 2080, 2012] payload_rows_per_dst=[3892, 4118, 4192, 4122, 4114, 4146, 4160, 4024] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3892, 4118, 4192, 4122, 4114, 4146, 4160, 4024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=28408 dup_rows=92 dup_rows_remote=68 dedup_ratio=0.002808 router_tokens_per_expert=[2072, 2141, 2040, 2066, 2000, 2034, 1973, 2058] payload_rows_per_dst=[4010, 3908, 4138, 4182, 4112, 4054, 4004, 4268] dup_rows_per_dst=[0, 0, 6, 18, 32, 6, 6, 24] combine_rows_per_src=[4010, 3908, 4138, 4182, 4112, 4054, 4004, 4268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235323552 ingress_bytes=1644494848 link_bytes=1644494848 token_bytes=470532256 weight_movement_bytes=1409286144 total_nvlink_bytes=1879818400 weight_to_token_ratio=2.995089 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208484703e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2018, 2013, 2075, 2073, 2047, 2037, 2076] payload_rows_per_dst=[4090, 4036, 4026, 4150, 4146, 4094, 4074, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4036, 4026, 4150, 4146, 4094, 4074, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=114920 combine_payload_bytes=235356160 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235471080 ingress_bytes=2349285376 link_bytes=2349285376 token_bytes=470827240 weight_movement_bytes=2113929216 total_nvlink_bytes=2584756456 weight_to_token_ratio=4.489819 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315501747e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234995712 dispatch_meta_bytes=114744 combine_payload_bytes=234995712 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235110456 ingress_bytes=3053568000 link_bytes=3053568000 token_bytes=470106168 weight_movement_bytes=2818572288 total_nvlink_bytes=3288678456 weight_to_token_ratio=5.995608 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709870618e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234668032 dispatch_meta_bytes=114584 combine_payload_bytes=234668032 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234782616 ingress_bytes=2348597248 link_bytes=2348597248 token_bytes=469450648 weight_movement_bytes=2113929216 total_nvlink_bytes=2583379864 weight_to_token_ratio=4.502985 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315116424e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=232718336 dispatch_meta_bytes=113904 combine_payload_bytes=232718336 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232832240 ingress_bytes=2346647552 link_bytes=2346647552 token_bytes=465550576 weight_movement_bytes=2113929216 total_nvlink_bytes=2579479792 weight_to_token_ratio=4.540708 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314024675e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28718 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2127, 2118, 2050, 2026, 2000, 2025, 2013, 2025] payload_rows_per_dst=[4254, 4236, 4100, 4052, 4000, 4050, 4026, 4050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4254, 4236, 4100, 4052, 4000, 4050, 4026, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=28544 dup_rows=66 dup_rows_remote=60 dedup_ratio=0.002014 router_tokens_per_expert=[2066, 2052, 2065, 2008, 2057, 2021, 2052, 2063] payload_rows_per_dst=[4158, 4104, 4430, 3846, 4114, 4090, 4104, 3856] dup_rows_per_dst=[6, 0, 50, 0, 0, 10, 0, 0] combine_rows_per_src=[4158, 4104, 4430, 3846, 4114, 4090, 4104, 3856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=235487472 ingress_bytes=4463230976 link_bytes=4463230976 token_bytes=470860016 weight_movement_bytes=4227858432 total_nvlink_bytes=4698718448 weight_to_token_ratio=8.979013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499223042e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28758 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2023, 2031, 2045, 2005, 2136, 2079, 2000] payload_rows_per_dst=[4130, 4046, 4062, 4090, 4010, 4272, 4158, 4000] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4046, 4062, 4090, 4010, 4272, 4158, 4000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235257856 dispatch_meta_bytes=114872 combine_payload_bytes=235257856 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235372728 ingress_bytes=2349187072 link_bytes=2349187072 token_bytes=470630584 weight_movement_bytes=2113929216 total_nvlink_bytes=2584559800 weight_to_token_ratio=4.491695 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315446700e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233832448 dispatch_meta_bytes=114416 combine_payload_bytes=233832448 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233946864 ingress_bytes=3757047808 link_bytes=3757047808 token_bytes=467779312 weight_movement_bytes=3523215360 total_nvlink_bytes=3990994672 weight_to_token_ratio=7.531790 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103789946e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28634 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 1952, 2067, 2037, 2056, 2052, 2084, 2070] payload_rows_per_dst=[4132, 3904, 4134, 4074, 4112, 4104, 4168, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 3904, 4134, 4074, 4112, 4104, 4168, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28686 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2066, 1964, 2119, 2053, 2004, 2041, 2082] payload_rows_per_dst=[4110, 4132, 3928, 4238, 4106, 4008, 4082, 4164] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4132, 3928, 4238, 4106, 4008, 4082, 4164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28790 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2077, 2041, 2083, 1989, 2064, 2014, 2039, 2077] payload_rows_per_dst=[4154, 4082, 4166, 3978, 4128, 4028, 4078, 4154] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4154, 4082, 4166, 3978, 4128, 4028, 4078, 4154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32650 payload_rows_remote=28452 dup_rows=118 dup_rows_remote=114 dedup_ratio=0.003601 router_tokens_per_expert=[2059, 2022, 2070, 2066, 2049, 2035, 1993, 2090] payload_rows_per_dst=[3908, 4022, 3824, 4400, 4036, 4058, 4204, 4198] dup_rows_per_dst=[0, 22, 0, 48, 6, 0, 38, 4] combine_rows_per_src=[3908, 4022, 3824, 4400, 4036, 4058, 4204, 4198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235585536 dispatch_meta_bytes=115032 combine_payload_bytes=235585536 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235700568 ingress_bytes=2349514752 link_bytes=2349514752 token_bytes=471286104 weight_movement_bytes=2113929216 total_nvlink_bytes=2585215320 weight_to_token_ratio=4.485448 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315630188e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28706 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2028, 2031, 2071, 2054, 2083, 2062, 2013, 2042] payload_rows_per_dst=[4056, 4062, 4142, 4108, 4166, 4124, 4026, 4084] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4056, 4062, 4142, 4108, 4166, 4124, 4026, 4084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234569728 dispatch_meta_bytes=114536 combine_payload_bytes=234569728 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234684264 ingress_bytes=2348498944 link_bytes=2348498944 token_bytes=469253992 weight_movement_bytes=2113929216 total_nvlink_bytes=2583183208 weight_to_token_ratio=4.504872 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315061377e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235847680 dispatch_meta_bytes=115160 combine_payload_bytes=235847680 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235962840 ingress_bytes=2349776896 link_bytes=2349776896 token_bytes=471810520 weight_movement_bytes=2113929216 total_nvlink_bytes=2585739736 weight_to_token_ratio=4.480462 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315776977e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233078784 dispatch_meta_bytes=114264 combine_payload_bytes=233078784 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233193048 ingress_bytes=3756294144 link_bytes=3756294144 token_bytes=466271832 weight_movement_bytes=3523215360 total_nvlink_bytes=3989487192 weight_to_token_ratio=7.556140 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103367925e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234995712 dispatch_meta_bytes=114744 combine_payload_bytes=234995712 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235110456 ingress_bytes=1644281856 link_bytes=1644281856 token_bytes=470106168 weight_movement_bytes=1409286144 total_nvlink_bytes=1879392312 weight_to_token_ratio=2.997804 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207292037e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2082, 2043, 2029, 2052, 2033, 2038, 2011, 2096] payload_rows_per_dst=[4164, 4086, 4058, 4104, 4066, 4076, 4022, 4192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4164, 4086, 4058, 4104, 4066, 4076, 4022, 4192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32650 payload_rows_remote=28254 dup_rows=118 dup_rows_remote=70 dedup_ratio=0.003601 router_tokens_per_expert=[2056, 2109, 2082, 2001, 2028, 1985, 2059, 2064] payload_rows_per_dst=[4396, 3944, 4314, 4122, 4056, 3956, 4042, 3820] dup_rows_per_dst=[48, 0, 26, 20, 0, 24, 0, 0] combine_rows_per_src=[4396, 3944, 4314, 4122, 4056, 3956, 4042, 3820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235159552 dispatch_meta_bytes=114824 combine_payload_bytes=235159552 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235274376 ingress_bytes=2349088768 link_bytes=2349088768 token_bytes=470433928 weight_movement_bytes=2113929216 total_nvlink_bytes=2584363144 weight_to_token_ratio=4.493573 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315391654e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28620 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2080, 2072, 2028, 2074, 2000, 2038, 2046] payload_rows_per_dst=[4092, 4160, 4144, 4056, 4148, 4000, 4076, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4160, 4144, 4056, 4148, 4000, 4076, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235159632 ingress_bytes=3053617152 link_bytes=3053617152 token_bytes=470204496 weight_movement_bytes=2818572288 total_nvlink_bytes=3288776784 weight_to_token_ratio=5.994354 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709898141e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=231456768 dispatch_meta_bytes=113296 combine_payload_bytes=231456768 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=231570064 ingress_bytes=3050029056 link_bytes=3050029056 token_bytes=463026832 weight_movement_bytes=2818572288 total_nvlink_bytes=3281599120 weight_to_token_ratio=6.087276 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.707888957e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2106, 2054, 1978, 2068, 2051, 2028, 2058] payload_rows_per_dst=[4082, 4212, 4108, 3956, 4136, 4102, 4056, 4116] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4212, 4108, 3956, 4136, 4102, 4056, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2023, 2021, 2020, 1990, 2115, 2070, 2065] payload_rows_per_dst=[4160, 4046, 4042, 4040, 3980, 4230, 4140, 4130] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4046, 4042, 4040, 3980, 4230, 4140, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32654 payload_rows_remote=28336 dup_rows=114 dup_rows_remote=82 dedup_ratio=0.003479 router_tokens_per_expert=[2069, 2071, 2090, 2068, 2030, 2033, 1980, 2043] payload_rows_per_dst=[3906, 4060, 3738, 4516, 4042, 4098, 3976, 4318] dup_rows_per_dst=[0, 0, 0, 62, 6, 14, 0, 32] combine_rows_per_src=[3906, 4060, 3738, 4516, 4042, 4098, 3976, 4318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234455040 dispatch_meta_bytes=114480 combine_payload_bytes=234455040 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234569520 ingress_bytes=2348384256 link_bytes=2348384256 token_bytes=469024560 weight_movement_bytes=2113929216 total_nvlink_bytes=2582953776 weight_to_token_ratio=4.507076 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314997157e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28802 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1994, 2109, 2061, 2019, 2067, 2058, 1983, 2093] payload_rows_per_dst=[3988, 4218, 4122, 4038, 4134, 4116, 3966, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3988, 4218, 4122, 4038, 4134, 4116, 3966, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2057, 2059, 1937, 2079, 2109, 2118, 2009] payload_rows_per_dst=[4032, 4114, 4118, 3874, 4158, 4218, 4236, 4018] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4114, 4118, 3874, 4158, 4218, 4236, 4018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114640 combine_payload_bytes=234782720 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234897360 ingress_bytes=2348711936 link_bytes=2348711936 token_bytes=469680080 weight_movement_bytes=2113929216 total_nvlink_bytes=2583609296 weight_to_token_ratio=4.500785 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315180644e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235454688 ingress_bytes=1644625920 link_bytes=1644625920 token_bytes=470794464 weight_movement_bytes=1409286144 total_nvlink_bytes=1880080608 weight_to_token_ratio=2.993421 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209218651e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=232128512 dispatch_meta_bytes=113672 combine_payload_bytes=232128512 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232242184 ingress_bytes=3050700800 link_bytes=3050700800 token_bytes=464370696 weight_movement_bytes=2818572288 total_nvlink_bytes=3282942984 weight_to_token_ratio=6.069660 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708265105e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235945984 dispatch_meta_bytes=115208 combine_payload_bytes=235945984 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236061192 ingress_bytes=3054518272 link_bytes=3054518272 token_bytes=472007176 weight_movement_bytes=2818572288 total_nvlink_bytes=3290579464 weight_to_token_ratio=5.971461 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710402730e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2064, 2096, 1973, 2066, 2002, 2098, 2048] payload_rows_per_dst=[4074, 4128, 4192, 3946, 4132, 4004, 4196, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4128, 4192, 3946, 4132, 4004, 4196, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=28426 dup_rows=98 dup_rows_remote=58 dedup_ratio=0.002991 router_tokens_per_expert=[1999, 2057, 2023, 2070, 2065, 2067, 2031, 2072] payload_rows_per_dst=[4244, 4148, 4022, 4300, 4228, 3776, 3898, 4054] dup_rows_per_dst=[40, 6, 0, 20, 32, 0, 0, 0] combine_rows_per_src=[4244, 4148, 4022, 4300, 4228, 3776, 3898, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234848184 ingress_bytes=3757948928 link_bytes=3757948928 token_bytes=469581752 weight_movement_bytes=3523215360 total_nvlink_bytes=3992797112 weight_to_token_ratio=7.502880 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104294535e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2028, 2063, 2025, 1965, 2063, 2173, 2045, 2022] payload_rows_per_dst=[4056, 4126, 4050, 3930, 4126, 4346, 4090, 4044] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4056, 4126, 4050, 3930, 4126, 4346, 4090, 4044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=232865792 dispatch_meta_bytes=113936 combine_payload_bytes=232865792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232979728 ingress_bytes=1642151936 link_bytes=1642151936 token_bytes=465845520 weight_movement_bytes=1409286144 total_nvlink_bytes=1875131664 weight_to_token_ratio=3.025222 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195365374e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235634688 dispatch_meta_bytes=115056 combine_payload_bytes=235634688 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235749744 ingress_bytes=2349563904 link_bytes=2349563904 token_bytes=471384432 weight_movement_bytes=2113929216 total_nvlink_bytes=2585313648 weight_to_token_ratio=4.484512 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315657711e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2036, 2049, 2036, 2047, 2056, 2054, 2068, 2038] payload_rows_per_dst=[4072, 4098, 4072, 4094, 4112, 4108, 4136, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4072, 4098, 4072, 4094, 4112, 4108, 4136, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28720 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2053, 2073, 2096, 2024, 2057, 2069, 2000, 2012] payload_rows_per_dst=[4106, 4146, 4192, 4048, 4114, 4138, 4000, 4024] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4106, 4146, 4192, 4048, 4114, 4138, 4000, 4024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32700 payload_rows_remote=28570 dup_rows=68 dup_rows_remote=58 dedup_ratio=0.002075 router_tokens_per_expert=[2027, 2125, 2032, 2108, 2011, 2029, 2008, 2044] payload_rows_per_dst=[4054, 4016, 4096, 4216, 3912, 4150, 4126, 4130] dup_rows_per_dst=[0, 0, 12, 0, 0, 20, 26, 10] combine_rows_per_src=[4054, 4016, 4096, 4216, 3912, 4150, 4126, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114568 combine_payload_bytes=234635264 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234749832 ingress_bytes=3053207552 link_bytes=3053207552 token_bytes=469385096 weight_movement_bytes=2818572288 total_nvlink_bytes=3287957384 weight_to_token_ratio=6.004818 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709668782e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28620 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2025, 1967, 2070, 2051, 2084, 2074, 2076] payload_rows_per_dst=[4074, 4050, 3934, 4140, 4102, 4168, 4148, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4050, 3934, 4140, 4102, 4168, 4148, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28598 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2085, 2076, 1997, 1976, 2045, 2057, 2110] payload_rows_per_dst=[4076, 4170, 4152, 3994, 3952, 4090, 4114, 4220] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4170, 4152, 3994, 3952, 4090, 4114, 4220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235192416 ingress_bytes=2349006848 link_bytes=2349006848 token_bytes=470270048 weight_movement_bytes=2113929216 total_nvlink_bytes=2584199264 weight_to_token_ratio=4.495139 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315345783e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234045440 dispatch_meta_bytes=114512 combine_payload_bytes=234045440 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234159952 ingress_bytes=2347974656 link_bytes=2347974656 token_bytes=468205392 weight_movement_bytes=2113929216 total_nvlink_bytes=2582134608 weight_to_token_ratio=4.514961 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314767798e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235274240 dispatch_meta_bytes=114880 combine_payload_bytes=235274240 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235389120 ingress_bytes=1644560384 link_bytes=1644560384 token_bytes=470663360 weight_movement_bytes=1409286144 total_nvlink_bytes=1879949504 weight_to_token_ratio=2.994255 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208851677e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234455040 dispatch_meta_bytes=114480 combine_payload_bytes=234455040 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234569520 ingress_bytes=2348384256 link_bytes=2348384256 token_bytes=469024560 weight_movement_bytes=2113929216 total_nvlink_bytes=2582953776 weight_to_token_ratio=4.507076 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314997157e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32604 payload_rows_remote=28514 dup_rows=164 dup_rows_remote=160 dedup_ratio=0.005005 router_tokens_per_expert=[2027, 2076, 2055, 2071, 2046, 2030, 2059, 2020] payload_rows_per_dst=[4090, 4476, 4188, 4368, 4080, 3830, 4060, 3512] dup_rows_per_dst=[4, 76, 8, 32, 34, 0, 10, 0] combine_rows_per_src=[4090, 4476, 4188, 4368, 4080, 3830, 4060, 3512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28682 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2072, 2048, 2067, 2013, 2095, 2043, 2066, 1980] payload_rows_per_dst=[4144, 4096, 4134, 4026, 4190, 4086, 4132, 3960] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4144, 4096, 4134, 4026, 4190, 4086, 4132, 3960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2048, 1956, 2040, 2026, 2054, 2087, 2030, 2143] payload_rows_per_dst=[4096, 3912, 4080, 4052, 4108, 4174, 4060, 4286] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 3912, 4080, 4052, 4108, 4174, 4060, 4286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114392 combine_payload_bytes=234274816 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234389208 ingress_bytes=4462133248 link_bytes=4462133248 token_bytes=468664024 weight_movement_bytes=4227858432 total_nvlink_bytes=4696522456 weight_to_token_ratio=9.021086 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498608360e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233586688 dispatch_meta_bytes=114696 combine_payload_bytes=233586688 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233701384 ingress_bytes=2347515904 link_bytes=2347515904 token_bytes=467288072 weight_movement_bytes=2113929216 total_nvlink_bytes=2581217288 weight_to_token_ratio=4.523824 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314510916e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234962944 dispatch_meta_bytes=114728 combine_payload_bytes=234962944 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235077672 ingress_bytes=3053535232 link_bytes=3053535232 token_bytes=470040616 weight_movement_bytes=2818572288 total_nvlink_bytes=3288612904 weight_to_token_ratio=5.996444 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709852269e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28592 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1987, 2005, 2088, 2047, 2056, 1997, 2117, 2087] payload_rows_per_dst=[3974, 4010, 4176, 4094, 4112, 3994, 4234, 4174] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3974, 4010, 4176, 4094, 4112, 3994, 4234, 4174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2069, 2034, 2009, 2035, 2091, 2021, 2042, 2083] payload_rows_per_dst=[4138, 4068, 4018, 4070, 4182, 4042, 4084, 4166] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4138, 4068, 4018, 4070, 4182, 4042, 4084, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32758 payload_rows_remote=28560 dup_rows=10 dup_rows_remote=6 dedup_ratio=0.000305 router_tokens_per_expert=[2079, 2014, 2056, 2007, 2022, 2072, 2042, 2092] payload_rows_per_dst=[4136, 4028, 4102, 3968, 4030, 4160, 4136, 4198] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 4, 4] combine_rows_per_src=[4136, 4028, 4102, 3968, 4030, 4160, 4136, 4198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114640 combine_payload_bytes=234782720 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234897360 ingress_bytes=3757998080 link_bytes=3757998080 token_bytes=469680080 weight_movement_bytes=3523215360 total_nvlink_bytes=3992895440 weight_to_token_ratio=7.501309 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104322058e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2015, 2027, 2044, 2060, 2106, 2044, 2039] payload_rows_per_dst=[4098, 4030, 4054, 4088, 4120, 4212, 4088, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4030, 4054, 4088, 4120, 4212, 4088, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234225664 dispatch_meta_bytes=114368 combine_payload_bytes=234225664 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234340032 ingress_bytes=3052797952 link_bytes=3052797952 token_bytes=468565696 weight_movement_bytes=2818572288 total_nvlink_bytes=3287137984 weight_to_token_ratio=6.015319 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709439423e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28702 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1988, 2033, 2061, 2016, 2047, 2043, 2070, 2126] payload_rows_per_dst=[3976, 4066, 4122, 4032, 4094, 4086, 4140, 4252] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3976, 4066, 4122, 4032, 4094, 4086, 4140, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235208808 ingress_bytes=2349023232 link_bytes=2349023232 token_bytes=470302824 weight_movement_bytes=2113929216 total_nvlink_bytes=2584232040 weight_to_token_ratio=4.494826 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315354957e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114264 combine_payload_bytes=233963520 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234077784 ingress_bytes=4461821952 link_bytes=4461821952 token_bytes=468041304 weight_movement_bytes=4227858432 total_nvlink_bytes=4695899736 weight_to_token_ratio=9.033088 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498434048e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2104, 2059, 2034, 2013, 2015, 2077, 2043, 2039] payload_rows_per_dst=[4208, 4118, 4068, 4026, 4030, 4154, 4086, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4208, 4118, 4068, 4026, 4030, 4154, 4086, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28358 dup_rows=80 dup_rows_remote=58 dedup_ratio=0.002441 router_tokens_per_expert=[2050, 1981, 1998, 2130, 2019, 2042, 2125, 2039] payload_rows_per_dst=[4330, 4260, 4074, 3906, 3710, 4084, 4246, 4078] dup_rows_per_dst=[22, 46, 12, 0, 0, 0, 0, 0] combine_rows_per_src=[4330, 4260, 4074, 3906, 3710, 4084, 4246, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235061280 ingress_bytes=2348875776 link_bytes=2348875776 token_bytes=470007840 weight_movement_bytes=2113929216 total_nvlink_bytes=2583937056 weight_to_token_ratio=4.497647 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315272388e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28552 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2023, 2006, 2090, 2100, 2108, 1961, 2027, 2069] payload_rows_per_dst=[4046, 4012, 4180, 4200, 4216, 3922, 4054, 4138] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4046, 4012, 4180, 4200, 4216, 3922, 4054, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235126784 dispatch_meta_bytes=114808 combine_payload_bytes=235126784 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235241592 ingress_bytes=1644412928 link_bytes=1644412928 token_bytes=470368376 weight_movement_bytes=1409286144 total_nvlink_bytes=1879654520 weight_to_token_ratio=2.996133 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208025985e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234520344 ingress_bytes=2348335104 link_bytes=2348335104 token_bytes=468926232 weight_movement_bytes=2113929216 total_nvlink_bytes=2582855448 weight_to_token_ratio=4.508021 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314969634e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232308736 dispatch_meta_bytes=113664 combine_payload_bytes=232308736 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232422400 ingress_bytes=2346237952 link_bytes=2346237952 token_bytes=464731136 weight_movement_bytes=2113929216 total_nvlink_bytes=2578660352 weight_to_token_ratio=4.548714 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313795316e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28522 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1999, 2051, 2040, 2123, 2038, 2046, 2069, 2018] payload_rows_per_dst=[3998, 4102, 4080, 4246, 4076, 4092, 4138, 4036] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3998, 4102, 4080, 4246, 4076, 4092, 4138, 4036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28640 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2011, 1997, 2064, 2066, 2044, 2055, 2050, 2097] payload_rows_per_dst=[4022, 3994, 4128, 4132, 4088, 4110, 4100, 4194] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4022, 3994, 4128, 4132, 4088, 4110, 4100, 4194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233897984 dispatch_meta_bytes=114208 combine_payload_bytes=233897984 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234012192 ingress_bytes=2347827200 link_bytes=2347827200 token_bytes=467910176 weight_movement_bytes=2113929216 total_nvlink_bytes=2581839392 weight_to_token_ratio=4.517810 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314685229e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32712 payload_rows_remote=28648 dup_rows=56 dup_rows_remote=34 dedup_ratio=0.001709 router_tokens_per_expert=[2066, 2139, 2012, 2052, 2099, 2049, 2019, 1948] payload_rows_per_dst=[4056, 4212, 3890, 4036, 4246, 4140, 4068, 4064] dup_rows_per_dst=[0, 0, 0, 0, 18, 10, 6, 22] combine_rows_per_src=[4056, 4212, 3890, 4036, 4246, 4140, 4068, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28682 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2025, 1987, 2066, 2055, 2084, 2044, 2043, 2080] payload_rows_per_dst=[4050, 3974, 4132, 4110, 4168, 4088, 4086, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4050, 3974, 4132, 4110, 4168, 4088, 4086, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233652224 dispatch_meta_bytes=114088 combine_payload_bytes=233652224 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233766312 ingress_bytes=3756867584 link_bytes=3756867584 token_bytes=467418536 weight_movement_bytes=3523215360 total_nvlink_bytes=3990633896 weight_to_token_ratio=7.537603 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103689028e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2017, 2058, 2072, 1985, 2047, 2085, 2035, 2085] payload_rows_per_dst=[4034, 4116, 4144, 3970, 4094, 4170, 4070, 4170] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4034, 4116, 4144, 3970, 4094, 4170, 4070, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234618880 dispatch_meta_bytes=114560 combine_payload_bytes=234618880 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234733440 ingress_bytes=2348548096 link_bytes=2348548096 token_bytes=469352320 weight_movement_bytes=2113929216 total_nvlink_bytes=2583281536 weight_to_token_ratio=4.503928 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315088901e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114728 combine_payload_bytes=234684416 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234799144 ingress_bytes=1643970560 link_bytes=1643970560 token_bytes=469483560 weight_movement_bytes=1409286144 total_nvlink_bytes=1878769704 weight_to_token_ratio=3.001780 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205548909e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234962944 dispatch_meta_bytes=114728 combine_payload_bytes=234962944 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235077672 ingress_bytes=1644249088 link_bytes=1644249088 token_bytes=470040616 weight_movement_bytes=1409286144 total_nvlink_bytes=1879326760 weight_to_token_ratio=2.998222 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207108549e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28558 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2025, 1965, 2027, 2119, 2051, 2105, 2044, 2048] payload_rows_per_dst=[4050, 3930, 4054, 4238, 4102, 4210, 4088, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4050, 3930, 4054, 4238, 4102, 4210, 4088, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32658 payload_rows_remote=28414 dup_rows=110 dup_rows_remote=84 dedup_ratio=0.003357 router_tokens_per_expert=[2055, 2038, 2024, 2038, 2016, 2059, 2057, 2097] payload_rows_per_dst=[4244, 4324, 4060, 4086, 3870, 4210, 4146, 3718] dup_rows_per_dst=[26, 52, 0, 0, 4, 16, 12, 0] combine_rows_per_src=[4244, 4324, 4060, 4086, 3870, 4210, 4146, 3718] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2006, 2045, 2041, 2127, 2065, 2004, 2023, 2073] payload_rows_per_dst=[4012, 4090, 4082, 4254, 4130, 4008, 4046, 4146] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4012, 4090, 4082, 4254, 4130, 4008, 4046, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234831792 ingress_bytes=2348646400 link_bytes=2348646400 token_bytes=469548976 weight_movement_bytes=2113929216 total_nvlink_bytes=2583478192 weight_to_token_ratio=4.502042 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315143947e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2072, 1987, 2108, 2034, 2075, 2087, 1981, 2040] payload_rows_per_dst=[4144, 3974, 4216, 4068, 4150, 4174, 3962, 4080] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4144, 3974, 4216, 4068, 4150, 4174, 3962, 4080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233947136 dispatch_meta_bytes=114232 combine_payload_bytes=233947136 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234061368 ingress_bytes=1643233280 link_bytes=1643233280 token_bytes=468008504 weight_movement_bytes=1409286144 total_nvlink_bytes=1877294648 weight_to_token_ratio=3.011240 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201420449e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234717048 ingress_bytes=4462460928 link_bytes=4462460928 token_bytes=469319544 weight_movement_bytes=4227858432 total_nvlink_bytes=4697177976 weight_to_token_ratio=9.008486 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498791847e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=232767488 dispatch_meta_bytes=113992 combine_payload_bytes=232767488 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232881480 ingress_bytes=1642053632 link_bytes=1642053632 token_bytes=465648968 weight_movement_bytes=1409286144 total_nvlink_bytes=1874935112 weight_to_token_ratio=3.026499 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.194814912e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2058, 2101, 2031, 2069, 2026, 2006, 2002] payload_rows_per_dst=[4182, 4116, 4202, 4062, 4138, 4052, 4012, 4004] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4116, 4202, 4062, 4138, 4052, 4012, 4004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=28552 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.001831 router_tokens_per_expert=[1988, 2053, 2052, 1981, 2073, 2076, 2084, 2077] payload_rows_per_dst=[3976, 4022, 3954, 4138, 4114, 4182, 4166, 4156] dup_rows_per_dst=[0, 0, 20, 38, 0, 2, 0, 0] combine_rows_per_src=[3976, 4022, 3954, 4138, 4114, 4182, 4166, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1997, 2034, 2011, 1993, 2118, 2022, 2082, 2127] payload_rows_per_dst=[3994, 4068, 4022, 3986, 4236, 4044, 4164, 4254] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3994, 4068, 4022, 3986, 4236, 4044, 4164, 4254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235225200 ingress_bytes=1644396544 link_bytes=1644396544 token_bytes=470335600 weight_movement_bytes=1409286144 total_nvlink_bytes=1879621744 weight_to_token_ratio=2.996342 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207934242e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2035, 2030, 1986, 2059, 2020, 2104, 2104] payload_rows_per_dst=[4092, 4070, 4060, 3972, 4118, 4040, 4208, 4208] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4070, 4060, 3972, 4118, 4040, 4208, 4208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234126936 ingress_bytes=3052584960 link_bytes=3052584960 token_bytes=468139608 weight_movement_bytes=2818572288 total_nvlink_bytes=3286711896 weight_to_token_ratio=6.020794 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709320156e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233897984 dispatch_meta_bytes=114448 combine_payload_bytes=233897984 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234012432 ingress_bytes=4461756416 link_bytes=4461756416 token_bytes=467910416 weight_movement_bytes=4227858432 total_nvlink_bytes=4695768848 weight_to_token_ratio=9.035615 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498397350e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2006, 1994, 2016, 2090, 2059, 2109, 2028, 2082] payload_rows_per_dst=[4012, 3988, 4032, 4180, 4118, 4218, 4056, 4164] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4012, 3988, 4032, 4180, 4118, 4218, 4056, 4164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234438384 ingress_bytes=3052896256 link_bytes=3052896256 token_bytes=468762352 weight_movement_bytes=2818572288 total_nvlink_bytes=3287334640 weight_to_token_ratio=6.012796 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709494469e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28788 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2041, 2081, 2094, 2035, 1990, 2030, 2059] payload_rows_per_dst=[4108, 4082, 4162, 4188, 4070, 3980, 4060, 4118] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4082, 4162, 4188, 4070, 3980, 4060, 4118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32636 payload_rows_remote=28362 dup_rows=132 dup_rows_remote=124 dedup_ratio=0.004028 router_tokens_per_expert=[2100, 2000, 2050, 1990, 2004, 2174, 2042, 2024] payload_rows_per_dst=[4274, 4260, 3852, 4100, 4250, 3868, 3990, 4042] dup_rows_per_dst=[8, 50, 0, 36, 38, 0, 0, 0] combine_rows_per_src=[4274, 4260, 3852, 4100, 4250, 3868, 3990, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235208808 ingress_bytes=2349023232 link_bytes=2349023232 token_bytes=470302824 weight_movement_bytes=2113929216 total_nvlink_bytes=2584232040 weight_to_token_ratio=4.494826 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315354957e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2107, 2041, 2056, 2049, 2001, 2037, 2065, 2028] payload_rows_per_dst=[4214, 4082, 4112, 4098, 4002, 4074, 4130, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4214, 4082, 4112, 4098, 4002, 4074, 4130, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234700800 dispatch_meta_bytes=114600 combine_payload_bytes=234700800 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234815400 ingress_bytes=2348630016 link_bytes=2348630016 token_bytes=469516200 weight_movement_bytes=2113929216 total_nvlink_bytes=2583445416 weight_to_token_ratio=4.502356 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315134772e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=232341504 dispatch_meta_bytes=113944 combine_payload_bytes=232341504 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232455448 ingress_bytes=3050913792 link_bytes=3050913792 token_bytes=464796952 weight_movement_bytes=2818572288 total_nvlink_bytes=3283369240 weight_to_token_ratio=6.064094 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708384372e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235831296 dispatch_meta_bytes=115152 combine_payload_bytes=235831296 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235946448 ingress_bytes=1645117440 link_bytes=1645117440 token_bytes=471777744 weight_movement_bytes=1409286144 total_nvlink_bytes=1881063888 weight_to_token_ratio=2.987182 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211970958e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28576 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2086, 2043, 2096, 2019, 2038, 2056, 2002, 2044] payload_rows_per_dst=[4172, 4086, 4192, 4038, 4076, 4112, 4004, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4172, 4086, 4192, 4038, 4076, 4112, 4004, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=28440 dup_rows=48 dup_rows_remote=36 dedup_ratio=0.001465 router_tokens_per_expert=[2051, 2030, 2115, 2067, 2055, 2038, 1984, 2044] payload_rows_per_dst=[3804, 4060, 4230, 4122, 4000, 4076, 4148, 4280] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 36, 12] combine_rows_per_src=[3804, 4060, 4230, 4122, 4000, 4076, 4148, 4280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28504 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2123, 2045, 1993, 2016, 2017, 2077, 2132, 1981] payload_rows_per_dst=[4246, 4090, 3986, 4032, 4034, 4154, 4264, 3962] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4246, 4090, 3986, 4032, 4034, 4154, 4264, 3962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234979320 ingress_bytes=2348793856 link_bytes=2348793856 token_bytes=469843960 weight_movement_bytes=2113929216 total_nvlink_bytes=2583773176 weight_to_token_ratio=4.499215 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315226516e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2037, 2111, 2022, 2077, 2053, 1967, 2088] payload_rows_per_dst=[4058, 4074, 4222, 4044, 4154, 4106, 3934, 4176] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4074, 4222, 4044, 4154, 4106, 3934, 4176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=232980480 dispatch_meta_bytes=113904 combine_payload_bytes=232980480 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=233094384 ingress_bytes=1642266624 link_bytes=1642266624 token_bytes=466074864 weight_movement_bytes=1409286144 total_nvlink_bytes=1875361008 weight_to_token_ratio=3.023733 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196007579e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234094592 dispatch_meta_bytes=114304 combine_payload_bytes=234094592 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234208896 ingress_bytes=2348023808 link_bytes=2348023808 token_bytes=468303488 weight_movement_bytes=2113929216 total_nvlink_bytes=2582232704 weight_to_token_ratio=4.514016 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314795321e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28842 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2048, 2100, 2084, 1963, 2056, 2045, 2059] payload_rows_per_dst=[4058, 4096, 4200, 4168, 3926, 4112, 4090, 4118] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4096, 4200, 4168, 3926, 4112, 4090, 4118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233504768 dispatch_meta_bytes=114016 combine_payload_bytes=233504768 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233618784 ingress_bytes=2347433984 link_bytes=2347433984 token_bytes=467123552 weight_movement_bytes=2113929216 total_nvlink_bytes=2581052768 weight_to_token_ratio=4.525418 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314465044e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32668 payload_rows_remote=28418 dup_rows=100 dup_rows_remote=66 dedup_ratio=0.003052 router_tokens_per_expert=[2035, 2026, 2018, 2053, 2097, 2054, 2104, 1997] payload_rows_per_dst=[4250, 4190, 3904, 4302, 4276, 3794, 3958, 3994] dup_rows_per_dst=[34, 32, 0, 22, 12, 0, 0, 0] combine_rows_per_src=[4250, 4190, 3904, 4302, 4276, 3794, 3958, 3994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235176024 ingress_bytes=1644347392 link_bytes=1644347392 token_bytes=470237272 weight_movement_bytes=1409286144 total_nvlink_bytes=1879523416 weight_to_token_ratio=2.996968 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207659011e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 2083, 2085, 2044, 2054, 2062, 1965, 2045] payload_rows_per_dst=[4092, 4166, 4170, 4088, 4108, 4124, 3930, 4090] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 4166, 4170, 4088, 4108, 4124, 3930, 4090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28630 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1959, 2128, 2091, 2069, 2028, 2055, 1999, 2055] payload_rows_per_dst=[3918, 4256, 4182, 4138, 4056, 4110, 3998, 4110] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3918, 4256, 4182, 4138, 4056, 4110, 3998, 4110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=236273664 dispatch_meta_bytes=115368 combine_payload_bytes=236273664 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=236389032 ingress_bytes=3759489024 link_bytes=3759489024 token_bytes=472662696 weight_movement_bytes=3523215360 total_nvlink_bytes=3995878056 weight_to_token_ratio=7.453974 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.105156925e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232800256 dispatch_meta_bytes=113936 combine_payload_bytes=232800256 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=232914192 ingress_bytes=1642086400 link_bytes=1642086400 token_bytes=465714448 weight_movement_bytes=1409286144 total_nvlink_bytes=1875000592 weight_to_token_ratio=3.026073 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.194998399e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32730 payload_rows_remote=28426 dup_rows=38 dup_rows_remote=12 dedup_ratio=0.001160 router_tokens_per_expert=[2052, 1996, 2116, 1986, 2036, 2061, 2077, 2060] payload_rows_per_dst=[4100, 3992, 4194, 4010, 3804, 4162, 4164, 4304] dup_rows_per_dst=[0, 0, 0, 4, 0, 4, 4, 26] combine_rows_per_src=[4100, 3992, 4194, 4010, 3804, 4162, 4164, 4304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=0 reduce_bytes=2818572288 egress_bytes=234766224 ingress_bytes=3053223936 link_bytes=3053223936 token_bytes=469417872 weight_movement_bytes=2818572288 total_nvlink_bytes=3287990160 weight_to_token_ratio=6.004399 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709677956e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 1999, 2082, 2034, 2032, 2074, 2140, 1993] payload_rows_per_dst=[4060, 3998, 4164, 4068, 4064, 4148, 4280, 3986] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 3998, 4164, 4068, 4064, 4148, 4280, 3986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2002, 2046, 2012, 2052, 2120, 2038, 2050] payload_rows_per_dst=[4128, 4004, 4092, 4024, 4104, 4240, 4076, 4100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4004, 4092, 4024, 4104, 4240, 4076, 4100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114520 combine_payload_bytes=234536960 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234651480 ingress_bytes=1643823104 link_bytes=1643823104 token_bytes=469188440 weight_movement_bytes=1409286144 total_nvlink_bytes=1878474584 weight_to_token_ratio=3.003668 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204723217e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7515667904 prefetch_bytes=19025362944 reduce_bytes=21843935232 weight_movement_bytes=40869298176 egress_bytes=3758751168 ingress_bytes=44626214912 link_bytes=44626214912 total_nvlink_bytes=48384966080 weight_to_token_ratio=5.437880 prefetch_share_of_ingress=0.426327 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=31 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.561801098e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7463001736 prefetch_bytes=16911433728 reduce_bytes=23253221376 weight_movement_bytes=40164655104 egress_bytes=3732414088 ingress_bytes=43895242752 link_bytes=43895242752 total_nvlink_bytes=47627656840 weight_to_token_ratio=5.381836 prefetch_share_of_ingress=0.385268 dedup_ratio=0.002834 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=33 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.536218979e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2020, 2029, 2008, 2100, 2047, 2113, 2023] payload_rows_per_dst=[4088, 4040, 4058, 4016, 4200, 4094, 4226, 4046] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4040, 4058, 4016, 4200, 4094, 4226, 4046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232865792 dispatch_meta_bytes=113752 combine_payload_bytes=232865792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232979544 ingress_bytes=1642151936 link_bytes=1642151936 token_bytes=465845336 weight_movement_bytes=1409286144 total_nvlink_bytes=1875131480 weight_to_token_ratio=3.025223 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195365374e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234438384 ingress_bytes=3757539328 link_bytes=3757539328 token_bytes=468762352 weight_movement_bytes=3523215360 total_nvlink_bytes=3991977712 weight_to_token_ratio=7.515995 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104065176e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7516061216 prefetch_bytes=19730006016 reduce_bytes=16206790656 weight_movement_bytes=35936796672 egress_bytes=3758947872 ingress_bytes=39693910016 link_bytes=39693910016 total_nvlink_bytes=43452857888 weight_to_token_ratio=4.781334 prefetch_share_of_ingress=0.497054 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=23 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.389183294e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235159632 ingress_bytes=3053617152 link_bytes=3053617152 token_bytes=470204496 weight_movement_bytes=2818572288 total_nvlink_bytes=3288776784 weight_to_token_ratio=5.994354 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709898141e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235454688 ingress_bytes=1644625920 link_bytes=1644625920 token_bytes=470794464 weight_movement_bytes=1409286144 total_nvlink_bytes=1880080608 weight_to_token_ratio=2.993421 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209218651e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7469162304 prefetch_bytes=16206790656 reduce_bytes=23253221376 weight_movement_bytes=39460012032 egress_bytes=3735494464 ingress_bytes=43193679872 link_bytes=43193679872 total_nvlink_bytes=46929174336 weight_to_token_ratio=5.283057 prefetch_share_of_ingress=0.375212 dedup_ratio=0.002163 prefetch_slots=46 unused_prefetch_slots=18 slot_utilization=0.718750 reduce_slots=33 remote_demand=23 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.511666108e-04 prefetch_bytes_per_gemm_flop=5.671953914e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 1987, 2032, 2075, 2107, 2035, 2078, 2032] payload_rows_per_dst=[4076, 3974, 4064, 4150, 4214, 4070, 4156, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 3974, 4064, 4150, 4214, 4070, 4156, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7515733456 prefetch_bytes=19025362944 reduce_bytes=16206790656 weight_movement_bytes=35232153600 egress_bytes=3758783952 ingress_bytes=38989103104 link_bytes=38989103104 total_nvlink_bytes=42747887056 weight_to_token_ratio=4.687786 prefetch_share_of_ingress=0.487966 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=23 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.364516891e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7520715408 prefetch_bytes=21139292160 reduce_bytes=16206790656 weight_movement_bytes=37346082816 egress_bytes=3761275536 ingress_bytes=41105522688 link_bytes=41105522688 total_nvlink_bytes=44866798224 weight_to_token_ratio=4.965762 prefetch_share_of_ingress=0.514269 dedup_ratio=0.000000 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=23 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016098 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.438586055e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235208808 ingress_bytes=3758309376 link_bytes=3758309376 token_bytes=470302824 weight_movement_bytes=3523215360 total_nvlink_bytes=3993518184 weight_to_token_ratio=7.491376 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104496371e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2092, 2037, 1967, 2088, 2091, 2048, 2003] payload_rows_per_dst=[4116, 4184, 4074, 3934, 4176, 4182, 4096, 4006] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4184, 4074, 3934, 4176, 4182, 4096, 4006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7516913392 prefetch_bytes=19730006016 reduce_bytes=19025362944 weight_movement_bytes=38755368960 egress_bytes=3759374064 ingress_bytes=42512908288 link_bytes=42512908288 total_nvlink_bytes=46272282352 weight_to_token_ratio=5.155756 prefetch_share_of_ingress=0.464094 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=27 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.487840879e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235176024 ingress_bytes=3053633536 link_bytes=3053633536 token_bytes=470237272 weight_movement_bytes=2818572288 total_nvlink_bytes=3288809560 weight_to_token_ratio=5.993936 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709907315e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7509407688 prefetch_bytes=21139292160 reduce_bytes=16911433728 weight_movement_bytes=38050725888 egress_bytes=3755620296 ingress_bytes=41804513280 link_bytes=41804513280 total_nvlink_bytes=45560133576 weight_to_token_ratio=5.067074 prefetch_share_of_ingress=0.505670 dedup_ratio=0.000000 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=24 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.463048902e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none - [2026-09-16 21:35:54] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 569.9 | throughput per GPU (TFLOP/s/GPU): 585.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.704499E+00 | loss scale: 1.0 | grad norm: 7.168 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 2098, 2060, 2027, 2046, 2084, 2047, 1987] payload_rows_per_dst=[4070, 4196, 4120, 4054, 4092, 4168, 4094, 3974] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 4196, 4120, 4054, 4092, 4168, 4094, 3974] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=28706 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.001587 router_tokens_per_expert=[2005, 2067, 2061, 2041, 2011, 2045, 2077, 2077] payload_rows_per_dst=[4010, 4134, 4310, 4222, 4124, 4108, 3978, 3830] dup_rows_per_dst=[0, 0, 18, 18, 12, 4, 0, 0] combine_rows_per_src=[4010, 4134, 4310, 4222, 4124, 4108, 3978, 3830] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32678 payload_rows_remote=28494 dup_rows=90 dup_rows_remote=88 dedup_ratio=0.002747 router_tokens_per_expert=[2085, 2012, 1976, 2045, 2080, 2062, 2054, 2070] payload_rows_per_dst=[3970, 3842, 3722, 4382, 4352, 4086, 4140, 4184] dup_rows_per_dst=[0, 6, 0, 40, 38, 0, 4, 2] combine_rows_per_src=[3970, 3842, 3722, 4382, 4352, 4086, 4140, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2042, 2004, 2015, 2031, 2023, 2112, 2093] payload_rows_per_dst=[4128, 4084, 4008, 4030, 4062, 4046, 4224, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4084, 4008, 4030, 4062, 4046, 4224, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28682 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2023, 2142, 2046, 1959, 2024, 2043, 2117] payload_rows_per_dst=[4060, 4046, 4284, 4092, 3918, 4048, 4086, 4234] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4046, 4284, 4092, 3918, 4048, 4086, 4234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234913792 dispatch_meta_bytes=114704 combine_payload_bytes=234913792 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235028496 ingress_bytes=1644199936 link_bytes=1644199936 token_bytes=469942288 weight_movement_bytes=1409286144 total_nvlink_bytes=1879228432 weight_to_token_ratio=2.998849 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206833319e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28786 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2097, 2133, 2024, 1991, 2000, 2019, 2042, 2078] payload_rows_per_dst=[4194, 4266, 4048, 3982, 4000, 4038, 4084, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4266, 4048, 3982, 4000, 4038, 4084, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2044, 2084, 1971, 2085, 2034, 2046, 2049] payload_rows_per_dst=[4142, 4088, 4168, 3942, 4170, 4068, 4092, 4098] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4088, 4168, 3942, 4170, 4068, 4092, 4098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235159552 dispatch_meta_bytes=115032 combine_payload_bytes=235159552 prefetch_bytes=0 reduce_bytes=4227858432 egress_bytes=235274584 ingress_bytes=4463017984 link_bytes=4463017984 token_bytes=470434136 weight_movement_bytes=4227858432 total_nvlink_bytes=4698292568 weight_to_token_ratio=8.987142 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499103776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2097, 2053, 2045, 2003, 2000, 2101, 2033, 2052] payload_rows_per_dst=[4194, 4106, 4090, 4006, 4000, 4202, 4066, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4106, 4090, 4006, 4000, 4202, 4066, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233422848 dispatch_meta_bytes=114328 combine_payload_bytes=233422848 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233537176 ingress_bytes=3756638208 link_bytes=3756638208 token_bytes=466960024 weight_movement_bytes=3523215360 total_nvlink_bytes=3990175384 weight_to_token_ratio=7.545004 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103560587e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235601920 dispatch_meta_bytes=115040 combine_payload_bytes=235601920 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235716960 ingress_bytes=2349531136 link_bytes=2349531136 token_bytes=471318880 weight_movement_bytes=2113929216 total_nvlink_bytes=2585248096 weight_to_token_ratio=4.485136 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315639362e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235814912 dispatch_meta_bytes=115144 combine_payload_bytes=235814912 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235930056 ingress_bytes=1645101056 link_bytes=1645101056 token_bytes=471744968 weight_movement_bytes=1409286144 total_nvlink_bytes=1881031112 weight_to_token_ratio=2.987390 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211879215e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234962944 dispatch_meta_bytes=114728 combine_payload_bytes=234962944 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235077672 ingress_bytes=3053535232 link_bytes=3053535232 token_bytes=470040616 weight_movement_bytes=2818572288 total_nvlink_bytes=3288612904 weight_to_token_ratio=5.996444 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709852269e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28738 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2032, 2018, 2101, 2115, 2015, 2076, 2050, 1977] payload_rows_per_dst=[4064, 4036, 4202, 4230, 4030, 4152, 4100, 3954] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4064, 4036, 4202, 4230, 4030, 4152, 4100, 3954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235225200 ingress_bytes=2349039616 link_bytes=2349039616 token_bytes=470335600 weight_movement_bytes=2113929216 total_nvlink_bytes=2584264816 weight_to_token_ratio=4.494512 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315364131e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234913752 ingress_bytes=2348728320 link_bytes=2348728320 token_bytes=469712856 weight_movement_bytes=2113929216 total_nvlink_bytes=2583642072 weight_to_token_ratio=4.500471 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315189818e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32740 payload_rows_remote=28730 dup_rows=28 dup_rows_remote=26 dedup_ratio=0.000854 router_tokens_per_expert=[1997, 2163, 2038, 2015, 2069, 1994, 2061, 2047] payload_rows_per_dst=[4010, 4332, 4142, 4030, 4120, 3932, 4122, 4052] dup_rows_per_dst=[2, 2, 24, 0, 0, 0, 0, 0] combine_rows_per_src=[4010, 4332, 4142, 4030, 4120, 3932, 4122, 4052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235421696 dispatch_meta_bytes=114952 combine_payload_bytes=235421696 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235536648 ingress_bytes=2349350912 link_bytes=2349350912 token_bytes=470958344 weight_movement_bytes=2113929216 total_nvlink_bytes=2584887560 weight_to_token_ratio=4.488569 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315538444e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=115024 combine_payload_bytes=235356160 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235471184 ingress_bytes=3053928448 link_bytes=3053928448 token_bytes=470827344 weight_movement_bytes=2818572288 total_nvlink_bytes=3289399632 weight_to_token_ratio=5.986424 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710072454e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32652 payload_rows_remote=28566 dup_rows=116 dup_rows_remote=104 dedup_ratio=0.003540 router_tokens_per_expert=[2081, 2061, 2001, 2162, 2059, 2018, 1981, 2021] payload_rows_per_dst=[3986, 3904, 3914, 4290, 4170, 4152, 4150, 4086] dup_rows_per_dst=[0, 0, 0, 30, 26, 24, 24, 12] combine_rows_per_src=[3986, 3904, 3914, 4290, 4170, 4152, 4150, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28720 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2112, 2033, 2024, 2057, 2055, 2019, 2016, 2068] payload_rows_per_dst=[4224, 4066, 4048, 4114, 4110, 4038, 4032, 4136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4224, 4066, 4048, 4114, 4110, 4038, 4032, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2053, 2047, 2047, 2017, 1984, 2002, 2104, 2130] payload_rows_per_dst=[4106, 4094, 4094, 4034, 3968, 4004, 4208, 4260] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4106, 4094, 4094, 4034, 3968, 4004, 4208, 4260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28848 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2067, 2048, 2096, 1960, 2033, 2094, 2008, 2078] payload_rows_per_dst=[4134, 4096, 4192, 3920, 4066, 4188, 4016, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4134, 4096, 4192, 3920, 4066, 4188, 4016, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114680 combine_payload_bytes=234012672 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234127352 ingress_bytes=2347941888 link_bytes=2347941888 token_bytes=468140024 weight_movement_bytes=2113929216 total_nvlink_bytes=2582069240 weight_to_token_ratio=4.515592 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314749449e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2058, 2028, 2010, 2015, 2058, 2075, 2066] payload_rows_per_dst=[4148, 4116, 4056, 4020, 4030, 4116, 4150, 4132] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4116, 4056, 4020, 4030, 4116, 4150, 4132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235274240 dispatch_meta_bytes=114880 combine_payload_bytes=235274240 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235389120 ingress_bytes=2349203456 link_bytes=2349203456 token_bytes=470663360 weight_movement_bytes=2113929216 total_nvlink_bytes=2584592576 weight_to_token_ratio=4.491383 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315455875e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2017, 2046, 2043, 2068, 2015, 2065, 2066] payload_rows_per_dst=[4128, 4034, 4092, 4086, 4136, 4030, 4130, 4132] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4034, 4092, 4086, 4136, 4030, 4130, 4132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32714 payload_rows_remote=28678 dup_rows=54 dup_rows_remote=34 dedup_ratio=0.001648 router_tokens_per_expert=[1986, 2034, 1976, 2078, 2041, 2115, 2021, 2133] payload_rows_per_dst=[4036, 4074, 4098, 4180, 4082, 4230, 4036, 3978] dup_rows_per_dst=[20, 0, 18, 16, 0, 0, 0, 0] combine_rows_per_src=[4036, 4074, 4098, 4180, 4082, 4230, 4036, 3978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 2061, 2048, 2050, 2124, 2081, 1983, 1990] payload_rows_per_dst=[4094, 4122, 4096, 4100, 4248, 4162, 3966, 3980] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 4122, 4096, 4100, 4248, 4162, 3966, 3980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114240 combine_payload_bytes=233963520 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234077760 ingress_bytes=1643249664 link_bytes=1643249664 token_bytes=468041280 weight_movement_bytes=1409286144 total_nvlink_bytes=1877327424 weight_to_token_ratio=3.011030 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201512192e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=236322816 dispatch_meta_bytes=115392 combine_payload_bytes=236322816 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236438208 ingress_bytes=3054895104 link_bytes=3054895104 token_bytes=472761024 weight_movement_bytes=2818572288 total_nvlink_bytes=3291333312 weight_to_token_ratio=5.961939 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710613741e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234831792 ingress_bytes=3053289472 link_bytes=3053289472 token_bytes=469548976 weight_movement_bytes=2818572288 total_nvlink_bytes=3288121264 weight_to_token_ratio=6.002723 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709714654e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234667872 ingress_bytes=2348482560 link_bytes=2348482560 token_bytes=469221216 weight_movement_bytes=2113929216 total_nvlink_bytes=2583150432 weight_to_token_ratio=4.505187 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315052203e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234930176 dispatch_meta_bytes=114848 combine_payload_bytes=234930176 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235045024 ingress_bytes=1644216320 link_bytes=1644216320 token_bytes=469975200 weight_movement_bytes=1409286144 total_nvlink_bytes=1879261344 weight_to_token_ratio=2.998639 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206925062e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114424 combine_payload_bytes=234340352 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234454776 ingress_bytes=2348269568 link_bytes=2348269568 token_bytes=468795128 weight_movement_bytes=2113929216 total_nvlink_bytes=2582724344 weight_to_token_ratio=4.509282 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314932936e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32726 payload_rows_remote=28610 dup_rows=42 dup_rows_remote=30 dedup_ratio=0.001282 router_tokens_per_expert=[2094, 2056, 2080, 2063, 2056, 2020, 1983, 2032] payload_rows_per_dst=[4130, 4112, 4160, 4082, 4112, 3850, 4164, 4116] dup_rows_per_dst=[0, 6, 0, 0, 2, 0, 22, 12] combine_rows_per_src=[4130, 4112, 4160, 4082, 4112, 3850, 4164, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28608 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1977, 2116, 2080, 2048, 2072, 2078, 2012, 2001] payload_rows_per_dst=[3954, 4232, 4160, 4096, 4144, 4156, 4024, 4002] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3954, 4232, 4160, 4096, 4144, 4156, 4024, 4002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2028, 2058, 2028, 2036, 2059, 2061, 2027, 2087] payload_rows_per_dst=[4056, 4116, 4056, 4072, 4118, 4122, 4054, 4174] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4056, 4116, 4056, 4072, 4118, 4122, 4054, 4174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2048, 2062, 2040, 2088, 2010, 2038, 2082, 2016] payload_rows_per_dst=[4096, 4124, 4080, 4176, 4020, 4076, 4164, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 4124, 4080, 4176, 4020, 4076, 4164, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28800 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1958, 2099, 2079, 2092, 1984, 2040, 2049, 2083] payload_rows_per_dst=[3916, 4198, 4158, 4184, 3968, 4080, 4098, 4166] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3916, 4198, 4158, 4184, 3968, 4080, 4098, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=28274 dup_rows=92 dup_rows_remote=42 dedup_ratio=0.002808 router_tokens_per_expert=[2066, 2030, 2058, 2021, 2040, 2147, 2036, 1986] payload_rows_per_dst=[4402, 4142, 3830, 4008, 4168, 4198, 3956, 3972] dup_rows_per_dst=[50, 14, 0, 0, 28, 0, 0, 0] combine_rows_per_src=[4402, 4142, 3830, 4008, 4168, 4198, 3956, 3972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234373120 dispatch_meta_bytes=114560 combine_payload_bytes=234373120 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234487680 ingress_bytes=3757588480 link_bytes=3757588480 token_bytes=468860800 weight_movement_bytes=3523215360 total_nvlink_bytes=3992076160 weight_to_token_ratio=7.514417 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104092699e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234356736 dispatch_meta_bytes=114432 combine_payload_bytes=234356736 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234471168 ingress_bytes=1643642880 link_bytes=1643642880 token_bytes=468827904 weight_movement_bytes=1409286144 total_nvlink_bytes=1878114048 weight_to_token_ratio=3.005978 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203714038e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235192416 ingress_bytes=3053649920 link_bytes=3053649920 token_bytes=470270048 weight_movement_bytes=2818572288 total_nvlink_bytes=3288842336 weight_to_token_ratio=5.993519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709916490e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2075, 2007, 2041, 2022, 2062, 2040, 2053, 2084] payload_rows_per_dst=[4150, 4014, 4082, 4044, 4124, 4080, 4106, 4168] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4150, 4014, 4082, 4044, 4124, 4080, 4106, 4168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2099, 2028, 2011, 2008, 2015, 2112, 2034, 2077] payload_rows_per_dst=[4198, 4056, 4022, 4016, 4030, 4224, 4068, 4154] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4198, 4056, 4022, 4016, 4030, 4224, 4068, 4154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235929600 dispatch_meta_bytes=115200 combine_payload_bytes=235929600 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236044800 ingress_bytes=1645215744 link_bytes=1645215744 token_bytes=471974400 weight_movement_bytes=1409286144 total_nvlink_bytes=1881260544 weight_to_token_ratio=2.985938 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212521420e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234438384 ingress_bytes=2348253184 link_bytes=2348253184 token_bytes=468762352 weight_movement_bytes=2113929216 total_nvlink_bytes=2582691568 weight_to_token_ratio=4.509597 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314923762e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=231620608 dispatch_meta_bytes=113264 combine_payload_bytes=231620608 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231733872 ingress_bytes=2345549824 link_bytes=2345549824 token_bytes=463354480 weight_movement_bytes=2113929216 total_nvlink_bytes=2577283696 weight_to_token_ratio=4.562229 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313409993e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235126848 ingress_bytes=2348941312 link_bytes=2348941312 token_bytes=470138944 weight_movement_bytes=2113929216 total_nvlink_bytes=2584068160 weight_to_token_ratio=4.496392 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315309085e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235323552 ingress_bytes=3053780992 link_bytes=3053780992 token_bytes=470532256 weight_movement_bytes=2818572288 total_nvlink_bytes=3289104544 weight_to_token_ratio=5.990179 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709989884e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28570 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1944, 2079, 2051, 2099, 2101, 2048, 1992, 2070] payload_rows_per_dst=[3888, 4158, 4102, 4198, 4202, 4096, 3984, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3888, 4158, 4102, 4198, 4202, 4096, 3984, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=28442 dup_rows=98 dup_rows_remote=68 dedup_ratio=0.002991 router_tokens_per_expert=[2063, 2110, 2135, 2011, 2023, 1997, 2043, 2002] payload_rows_per_dst=[3896, 3952, 4096, 4100, 4248, 4028, 4122, 4228] dup_rows_per_dst=[0, 0, 0, 22, 28, 6, 12, 30] combine_rows_per_src=[3896, 3952, 4096, 4100, 4248, 4028, 4122, 4228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28596 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2090, 2070, 2086, 2021, 2010, 1986, 2113, 2008] payload_rows_per_dst=[4180, 4140, 4172, 4042, 4020, 3972, 4226, 4016] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4180, 4140, 4172, 4042, 4020, 3972, 4226, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2042, 2034, 2106, 2006, 2079, 2070, 2003] payload_rows_per_dst=[4088, 4084, 4068, 4212, 4012, 4158, 4140, 4006] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4084, 4068, 4212, 4012, 4158, 4140, 4006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32730 payload_rows_remote=28420 dup_rows=38 dup_rows_remote=32 dedup_ratio=0.001160 router_tokens_per_expert=[2112, 2035, 2020, 1955, 2073, 1950, 2146, 2093] payload_rows_per_dst=[4310, 4078, 4082, 3910, 4192, 3788, 4218, 4152] dup_rows_per_dst=[6, 12, 4, 0, 16, 0, 0, 0] combine_rows_per_src=[4310, 4078, 4082, 3910, 4192, 3788, 4218, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28556 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2076, 2037, 2034, 2003, 2061, 2106, 2013] payload_rows_per_dst=[4108, 4152, 4074, 4068, 4006, 4122, 4212, 4026] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4152, 4074, 4068, 4006, 4122, 4212, 4026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234258432 dispatch_meta_bytes=114384 combine_payload_bytes=234258432 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234372816 ingress_bytes=4462116864 link_bytes=4462116864 token_bytes=468631248 weight_movement_bytes=4227858432 total_nvlink_bytes=4696489680 weight_to_token_ratio=9.021717 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498599186e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234045440 dispatch_meta_bytes=114280 combine_payload_bytes=234045440 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234159720 ingress_bytes=3052617728 link_bytes=3052617728 token_bytes=468205160 weight_movement_bytes=2818572288 total_nvlink_bytes=3286777448 weight_to_token_ratio=6.019951 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709338505e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=232996864 dispatch_meta_bytes=114040 combine_payload_bytes=232996864 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233110904 ingress_bytes=1642283008 link_bytes=1642283008 token_bytes=466107768 weight_movement_bytes=1409286144 total_nvlink_bytes=1875393912 weight_to_token_ratio=3.023520 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196099322e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235684176 ingress_bytes=1644855296 link_bytes=1644855296 token_bytes=471253328 weight_movement_bytes=1409286144 total_nvlink_bytes=1880539472 weight_to_token_ratio=2.990507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210503061e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=232816640 dispatch_meta_bytes=113808 combine_payload_bytes=232816640 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=232930448 ingress_bytes=3051388928 link_bytes=3051388928 token_bytes=465747088 weight_movement_bytes=2818572288 total_nvlink_bytes=3284319376 weight_to_token_ratio=6.051723 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708650428e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2027, 2057, 2105, 2064, 2044, 2065, 2008, 2014] payload_rows_per_dst=[4054, 4114, 4210, 4128, 4088, 4130, 4016, 4028] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4054, 4114, 4210, 4128, 4088, 4130, 4016, 4028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28870 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2137, 1949, 2063, 2009, 2006, 2068, 2071, 2081] payload_rows_per_dst=[4274, 3898, 4126, 4018, 4012, 4136, 4142, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4274, 3898, 4126, 4018, 4012, 4136, 4142, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114224 combine_payload_bytes=233930752 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234044976 ingress_bytes=2347859968 link_bytes=2347859968 token_bytes=467975728 weight_movement_bytes=2113929216 total_nvlink_bytes=2581904944 weight_to_token_ratio=4.517177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314703578e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234717048 ingress_bytes=2348531712 link_bytes=2348531712 token_bytes=469319544 weight_movement_bytes=2113929216 total_nvlink_bytes=2583248760 weight_to_token_ratio=4.504243 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315079726e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=236503040 dispatch_meta_bytes=115480 combine_payload_bytes=236503040 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=236618520 ingress_bytes=3759718400 link_bytes=3759718400 token_bytes=473121560 weight_movement_bytes=3523215360 total_nvlink_bytes=3996336920 weight_to_token_ratio=7.446744 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.105285366e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28666 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2048, 2051, 2045, 2092, 2091, 2030, 1978] payload_rows_per_dst=[4098, 4096, 4102, 4090, 4184, 4182, 4060, 3956] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4096, 4102, 4090, 4184, 4182, 4060, 3956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2068, 2093, 2071, 2040, 1991, 2089, 2041, 1991] payload_rows_per_dst=[4136, 4186, 4142, 4080, 3982, 4178, 4082, 3982] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4136, 4186, 4142, 4080, 3982, 4178, 4082, 3982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32712 payload_rows_remote=28464 dup_rows=56 dup_rows_remote=40 dedup_ratio=0.001709 router_tokens_per_expert=[2057, 2067, 2036, 2078, 2019, 2051, 2014, 2062] payload_rows_per_dst=[4058, 4000, 3934, 4126, 4088, 4148, 4110, 4248] dup_rows_per_dst=[0, 0, 0, 4, 12, 12, 12, 16] combine_rows_per_src=[4058, 4000, 3934, 4126, 4088, 4148, 4110, 4248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32696 payload_rows_remote=28434 dup_rows=72 dup_rows_remote=64 dedup_ratio=0.002197 router_tokens_per_expert=[2099, 1995, 2038, 2021, 2027, 2078, 2093, 2033] payload_rows_per_dst=[4262, 3992, 4184, 4040, 4176, 4084, 4342, 3616] dup_rows_per_dst=[8, 0, 16, 0, 24, 0, 24, 0] combine_rows_per_src=[4262, 3992, 4184, 4040, 4176, 4084, 4342, 3616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2089, 1975, 1997, 2009, 2134, 2056, 2083] payload_rows_per_dst=[4082, 4178, 3950, 3994, 4018, 4268, 4112, 4166] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4178, 3950, 3994, 4018, 4268, 4112, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2026, 2037, 2059, 2061, 2086, 2009, 2036, 2070] payload_rows_per_dst=[4052, 4074, 4118, 4122, 4172, 4018, 4072, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4052, 4074, 4118, 4122, 4172, 4018, 4072, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235126848 ingress_bytes=1644298240 link_bytes=1644298240 token_bytes=470138944 weight_movement_bytes=1409286144 total_nvlink_bytes=1879425088 weight_to_token_ratio=2.997595 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207383780e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234831872 dispatch_meta_bytes=114664 combine_payload_bytes=234831872 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234946536 ingress_bytes=2348761088 link_bytes=2348761088 token_bytes=469778408 weight_movement_bytes=2113929216 total_nvlink_bytes=2583707624 weight_to_token_ratio=4.499843 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315208167e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28546 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2006, 2094, 2057, 2045, 1998, 2111, 2017, 2056] payload_rows_per_dst=[4012, 4188, 4114, 4090, 3996, 4222, 4034, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4012, 4188, 4114, 4090, 3996, 4222, 4034, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=232931328 dispatch_meta_bytes=113992 combine_payload_bytes=232931328 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233045320 ingress_bytes=3756146688 link_bytes=3756146688 token_bytes=465976648 weight_movement_bytes=3523215360 total_nvlink_bytes=3989192008 weight_to_token_ratio=7.560927 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103285356e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233177088 dispatch_meta_bytes=114016 combine_payload_bytes=233177088 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233291104 ingress_bytes=2347106304 link_bytes=2347106304 token_bytes=466468192 weight_movement_bytes=2113929216 total_nvlink_bytes=2580397408 weight_to_token_ratio=4.531776 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314281557e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235635000 ingress_bytes=1644806144 link_bytes=1644806144 token_bytes=471155000 weight_movement_bytes=1409286144 total_nvlink_bytes=1880441144 weight_to_token_ratio=2.991131 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210227831e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235192416 ingress_bytes=3053649920 link_bytes=3053649920 token_bytes=470270048 weight_movement_bytes=2818572288 total_nvlink_bytes=3288842336 weight_to_token_ratio=5.993519 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709916490e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28718 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2004, 2025, 2102, 2067, 2057, 2031, 2020, 2078] payload_rows_per_dst=[4008, 4050, 4204, 4134, 4114, 4062, 4040, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4008, 4050, 4204, 4134, 4114, 4062, 4040, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233848832 dispatch_meta_bytes=114184 combine_payload_bytes=233848832 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233963016 ingress_bytes=2347778048 link_bytes=2347778048 token_bytes=467811848 weight_movement_bytes=2113929216 total_nvlink_bytes=2581741064 weight_to_token_ratio=4.518759 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314657706e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28796 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2090, 2046, 2059, 1986, 2022, 2114, 2014, 2053] payload_rows_per_dst=[4180, 4092, 4118, 3972, 4044, 4228, 4028, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4180, 4092, 4118, 3972, 4044, 4228, 4028, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2077, 2034, 2102, 2046, 2068, 1948, 2055] payload_rows_per_dst=[4108, 4154, 4068, 4204, 4092, 4136, 3896, 4110] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4154, 4068, 4204, 4092, 4136, 3896, 4110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235257856 dispatch_meta_bytes=114872 combine_payload_bytes=235257856 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235372728 ingress_bytes=3758473216 link_bytes=3758473216 token_bytes=470630584 weight_movement_bytes=3523215360 total_nvlink_bytes=3993845944 weight_to_token_ratio=7.486159 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104588115e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28382 dup_rows=84 dup_rows_remote=44 dedup_ratio=0.002563 router_tokens_per_expert=[2095, 2076, 2067, 2044, 2067, 1950, 2024, 2061] payload_rows_per_dst=[3920, 3964, 4166, 4176, 4184, 3880, 4092, 4302] dup_rows_per_dst=[0, 0, 8, 18, 4, 4, 10, 40] combine_rows_per_src=[3920, 3964, 4166, 4176, 4184, 3880, 4092, 4302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32694 payload_rows_remote=28500 dup_rows=74 dup_rows_remote=42 dedup_ratio=0.002258 router_tokens_per_expert=[2035, 2055, 2122, 2028, 2020, 2002, 2065, 2057] payload_rows_per_dst=[4194, 4174, 4432, 4056, 3970, 3930, 3934, 4004] dup_rows_per_dst=[32, 14, 28, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4174, 4432, 4056, 3970, 3930, 3934, 4004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28850 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2033, 2032, 2090, 1959, 2083, 2062, 2061] payload_rows_per_dst=[4128, 4066, 4064, 4180, 3918, 4166, 4124, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4066, 4064, 4180, 3918, 4166, 4124, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28784 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2031, 2056, 2053, 2011, 2058, 1992, 2128] payload_rows_per_dst=[4110, 4062, 4112, 4106, 4022, 4116, 3984, 4256] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4062, 4112, 4106, 4022, 4116, 3984, 4256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235896832 dispatch_meta_bytes=115184 combine_payload_bytes=235896832 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=236012016 ingress_bytes=1645182976 link_bytes=1645182976 token_bytes=471908848 weight_movement_bytes=1409286144 total_nvlink_bytes=1881194992 weight_to_token_ratio=2.986352 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212337932e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235225200 ingress_bytes=1644396544 link_bytes=1644396544 token_bytes=470335600 weight_movement_bytes=1409286144 total_nvlink_bytes=1879621744 weight_to_token_ratio=2.996342 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207934242e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2000, 2092, 1991, 2058, 2076, 2006, 2123, 2038] payload_rows_per_dst=[4000, 4184, 3982, 4116, 4152, 4012, 4246, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4000, 4184, 3982, 4116, 4152, 4012, 4246, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=232505344 dispatch_meta_bytes=113704 combine_payload_bytes=232505344 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=232619048 ingress_bytes=3755720704 link_bytes=3755720704 token_bytes=465124392 weight_movement_bytes=3523215360 total_nvlink_bytes=3988339752 weight_to_token_ratio=7.574781 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103046823e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114168 combine_payload_bytes=233472000 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233586168 ingress_bytes=2347401216 link_bytes=2347401216 token_bytes=467058168 weight_movement_bytes=2113929216 total_nvlink_bytes=2580987384 weight_to_token_ratio=4.526051 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314446696e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=236339200 dispatch_meta_bytes=115400 combine_payload_bytes=236339200 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236454600 ingress_bytes=3054911488 link_bytes=3054911488 token_bytes=472793800 weight_movement_bytes=2818572288 total_nvlink_bytes=3291366088 weight_to_token_ratio=5.961525 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710622915e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235798528 dispatch_meta_bytes=115136 combine_payload_bytes=235798528 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235913664 ingress_bytes=1645084672 link_bytes=1645084672 token_bytes=471712192 weight_movement_bytes=1409286144 total_nvlink_bytes=1880998336 weight_to_token_ratio=2.987597 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211787471e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28626 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2071, 2082, 2071, 2026, 2020, 2017, 2048] payload_rows_per_dst=[4098, 4142, 4164, 4142, 4052, 4040, 4034, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4142, 4164, 4142, 4052, 4040, 4034, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235684176 ingress_bytes=3054141440 link_bytes=3054141440 token_bytes=471253328 weight_movement_bytes=2818572288 total_nvlink_bytes=3289825616 weight_to_token_ratio=5.981013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710191720e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2013, 2003, 2133, 2087, 2060, 2017, 2033, 2038] payload_rows_per_dst=[4026, 4006, 4266, 4174, 4120, 4034, 4066, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4026, 4006, 4266, 4174, 4120, 4034, 4066, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234504192 dispatch_meta_bytes=114504 combine_payload_bytes=234504192 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234618696 ingress_bytes=4462362624 link_bytes=4462362624 token_bytes=469122888 weight_movement_bytes=4227858432 total_nvlink_bytes=4696981320 weight_to_token_ratio=9.012262 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498736801e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28758 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2050, 2029, 2005, 2105, 2044, 2077, 2021, 2053] payload_rows_per_dst=[4100, 4058, 4010, 4210, 4088, 4154, 4042, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4100, 4058, 4010, 4210, 4088, 4154, 4042, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32574 payload_rows_remote=28270 dup_rows=194 dup_rows_remote=154 dedup_ratio=0.005920 router_tokens_per_expert=[2016, 2069, 2023, 2076, 2011, 2066, 2055, 2068] payload_rows_per_dst=[3862, 3942, 3480, 4074, 4372, 4300, 4240, 4304] dup_rows_per_dst=[0, 0, 0, 36, 64, 30, 24, 40] combine_rows_per_src=[3862, 3942, 3480, 4074, 4372, 4300, 4240, 4304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32674 payload_rows_remote=28654 dup_rows=94 dup_rows_remote=92 dedup_ratio=0.002869 router_tokens_per_expert=[2004, 2018, 2105, 2063, 2025, 2026, 2083, 2060] payload_rows_per_dst=[4020, 4462, 4210, 4114, 4050, 4052, 3926, 3840] dup_rows_per_dst=[2, 92, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4020, 4462, 4210, 4114, 4050, 4052, 3926, 3840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28824 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2151, 1998, 2059, 2055, 1972, 2015, 2054, 2080] payload_rows_per_dst=[4302, 3996, 4118, 4110, 3944, 4030, 4108, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4302, 3996, 4118, 4110, 3944, 4030, 4108, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2070, 2040, 2077, 2038, 2032, 2060, 2048, 2019] payload_rows_per_dst=[4140, 4080, 4154, 4076, 4064, 4120, 4096, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4080, 4154, 4076, 4064, 4120, 4096, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234242048 dispatch_meta_bytes=114376 combine_payload_bytes=234242048 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234356424 ingress_bytes=3052814336 link_bytes=3052814336 token_bytes=468598472 weight_movement_bytes=2818572288 total_nvlink_bytes=3287170760 weight_to_token_ratio=6.014899 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709448597e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2114, 2066, 1971, 2043, 2023, 2069, 2042] payload_rows_per_dst=[4112, 4228, 4132, 3942, 4086, 4046, 4138, 4084] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4228, 4132, 3942, 4086, 4046, 4138, 4084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235585536 dispatch_meta_bytes=115032 combine_payload_bytes=235585536 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235700568 ingress_bytes=3054157824 link_bytes=3054157824 token_bytes=471286104 weight_movement_bytes=2818572288 total_nvlink_bytes=3289858392 weight_to_token_ratio=5.980597 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710200895e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=231587840 dispatch_meta_bytes=113696 combine_payload_bytes=231587840 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231701536 ingress_bytes=2345517056 link_bytes=2345517056 token_bytes=463289376 weight_movement_bytes=2113929216 total_nvlink_bytes=2577218592 weight_to_token_ratio=4.562870 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313391645e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114984 combine_payload_bytes=234733568 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=234848552 ingress_bytes=5167235072 link_bytes=5167235072 token_bytes=469582120 weight_movement_bytes=4932501504 total_nvlink_bytes=5402083624 weight_to_token_ratio=10.504023 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893435949e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=236126208 dispatch_meta_bytes=115296 combine_payload_bytes=236126208 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236241504 ingress_bytes=1645412352 link_bytes=1645412352 token_bytes=472367712 weight_movement_bytes=1409286144 total_nvlink_bytes=1881653856 weight_to_token_ratio=2.983451 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213622342e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234995712 ingress_bytes=3053453312 link_bytes=3053453312 token_bytes=469876736 weight_movement_bytes=2818572288 total_nvlink_bytes=3288449024 weight_to_token_ratio=5.998536 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709806397e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28568 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2087, 2100, 2101, 2010, 2043, 1994, 2052, 1997] payload_rows_per_dst=[4174, 4200, 4202, 4020, 4086, 3988, 4104, 3994] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4174, 4200, 4202, 4020, 4086, 3988, 4104, 3994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235405512 ingress_bytes=2349219840 link_bytes=2349219840 token_bytes=470696136 weight_movement_bytes=2113929216 total_nvlink_bytes=2584625352 weight_to_token_ratio=4.491070 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315465049e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28690 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2093, 2097, 2039, 2005, 2028, 2012, 2061] payload_rows_per_dst=[4098, 4186, 4194, 4078, 4010, 4056, 4024, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4186, 4194, 4078, 4010, 4056, 4024, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234029056 dispatch_meta_bytes=114272 combine_payload_bytes=234029056 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234143328 ingress_bytes=2347958272 link_bytes=2347958272 token_bytes=468172384 weight_movement_bytes=2113929216 total_nvlink_bytes=2582101600 weight_to_token_ratio=4.515280 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314758624e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32714 payload_rows_remote=28640 dup_rows=54 dup_rows_remote=36 dedup_ratio=0.001648 router_tokens_per_expert=[2014, 1994, 2057, 2088, 2085, 2054, 2077, 2015] payload_rows_per_dst=[3800, 3988, 4120, 4080, 4344, 4154, 4154, 4074] dup_rows_per_dst=[0, 0, 6, 0, 26, 4, 0, 18] combine_rows_per_src=[3800, 3988, 4120, 4080, 4344, 4154, 4154, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2043, 2045, 2027, 2053, 2092, 2041, 2087, 1996] payload_rows_per_dst=[4086, 4090, 4054, 4106, 4184, 4082, 4174, 3992] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4086, 4090, 4054, 4106, 4184, 4082, 4174, 3992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32646 payload_rows_remote=28444 dup_rows=122 dup_rows_remote=122 dedup_ratio=0.003723 router_tokens_per_expert=[2100, 2070, 2059, 1987, 2062, 2033, 1986, 2087] payload_rows_per_dst=[4202, 4140, 4340, 4204, 3842, 3800, 4132, 3986] dup_rows_per_dst=[0, 0, 44, 50, 0, 0, 28, 0] combine_rows_per_src=[4202, 4140, 4340, 4204, 3842, 3800, 4132, 3986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235028480 dispatch_meta_bytes=114760 combine_payload_bytes=235028480 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235143240 ingress_bytes=3758243840 link_bytes=3758243840 token_bytes=470171720 weight_movement_bytes=3523215360 total_nvlink_bytes=3993387080 weight_to_token_ratio=7.493465 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104459674e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2070, 2072, 2013, 2045, 2062, 2020, 2050, 2052] payload_rows_per_dst=[4140, 4144, 4026, 4090, 4124, 4040, 4100, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4144, 4026, 4090, 4124, 4040, 4100, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28774 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2086, 2041, 1963, 2060, 2054, 2090, 1997, 2093] payload_rows_per_dst=[4172, 4082, 3926, 4120, 4108, 4180, 3994, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4172, 4082, 3926, 4120, 4108, 4180, 3994, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28742 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 1959, 2094, 2057, 2018, 2013, 2118, 2064] payload_rows_per_dst=[4122, 3918, 4188, 4114, 4036, 4026, 4236, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 3918, 4188, 4114, 4036, 4026, 4236, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234618880 dispatch_meta_bytes=114704 combine_payload_bytes=234618880 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234733584 ingress_bytes=3053191168 link_bytes=3053191168 token_bytes=469352464 weight_movement_bytes=2818572288 total_nvlink_bytes=3287924752 weight_to_token_ratio=6.005236 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709659608e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235225088 dispatch_meta_bytes=114856 combine_payload_bytes=235225088 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235339944 ingress_bytes=2349154304 link_bytes=2349154304 token_bytes=470565032 weight_movement_bytes=2113929216 total_nvlink_bytes=2584494248 weight_to_token_ratio=4.492321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315428352e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233013248 dispatch_meta_bytes=114264 combine_payload_bytes=233013248 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233127512 ingress_bytes=3051585536 link_bytes=3051585536 token_bytes=466140760 weight_movement_bytes=2818572288 total_nvlink_bytes=3284713048 weight_to_token_ratio=6.046612 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708760521e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235716608 dispatch_meta_bytes=115096 combine_payload_bytes=235716608 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235831704 ingress_bytes=2349645824 link_bytes=2349645824 token_bytes=471548312 weight_movement_bytes=2113929216 total_nvlink_bytes=2585477528 weight_to_token_ratio=4.482954 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315703582e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234766224 ingress_bytes=3757867008 link_bytes=3757867008 token_bytes=469417872 weight_movement_bytes=3523215360 total_nvlink_bytes=3992633232 weight_to_token_ratio=7.505499 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104248663e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2083, 1977, 1976, 2023, 2033, 2092, 2076, 2124] payload_rows_per_dst=[4166, 3954, 3952, 4046, 4066, 4184, 4152, 4248] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4166, 3954, 3952, 4046, 4066, 4184, 4152, 4248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235454464 dispatch_meta_bytes=114968 combine_payload_bytes=235454464 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235569432 ingress_bytes=1644740608 link_bytes=1644740608 token_bytes=471023896 weight_movement_bytes=1409286144 total_nvlink_bytes=1880310040 weight_to_token_ratio=2.991963 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209860856e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2079, 1999, 2085, 2058, 2103, 2061, 1998, 2001] payload_rows_per_dst=[4158, 3998, 4170, 4116, 4206, 4122, 3996, 4002] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4158, 3998, 4170, 4116, 4206, 4122, 3996, 4002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=236044288 dispatch_meta_bytes=115256 combine_payload_bytes=236044288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236159544 ingress_bytes=1645330432 link_bytes=1645330432 token_bytes=472203832 weight_movement_bytes=1409286144 total_nvlink_bytes=1881489976 weight_to_token_ratio=2.984487 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213163624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=28556 dup_rows=34 dup_rows_remote=26 dedup_ratio=0.001038 router_tokens_per_expert=[2025, 2158, 2018, 2036, 1990, 2114, 1994, 2049] payload_rows_per_dst=[3936, 4156, 4062, 4096, 3938, 4380, 3988, 4178] dup_rows_per_dst=[0, 0, 4, 6, 2, 14, 0, 8] combine_rows_per_src=[3936, 4156, 4062, 4096, 3938, 4380, 3988, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28940 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2142, 2031, 1914, 2089, 2088, 2094, 2019, 2007] payload_rows_per_dst=[4284, 4062, 3828, 4178, 4176, 4188, 4038, 4014] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4284, 4062, 3828, 4178, 4176, 4188, 4038, 4014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234831792 ingress_bytes=1644003328 link_bytes=1644003328 token_bytes=469548976 weight_movement_bytes=1409286144 total_nvlink_bytes=1878835120 weight_to_token_ratio=3.001361 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205732396e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32736 payload_rows_remote=28334 dup_rows=32 dup_rows_remote=22 dedup_ratio=0.000977 router_tokens_per_expert=[2170, 2022, 2042, 2064, 2100, 2012, 2005, 1969] payload_rows_per_dst=[4402, 4044, 4148, 4142, 4028, 4030, 4010, 3932] dup_rows_per_dst=[10, 0, 14, 8, 0, 0, 0, 0] combine_rows_per_src=[4402, 4044, 4148, 4142, 4028, 4030, 4010, 3932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28636 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2008, 2030, 2010, 2044, 2081, 2082, 2066, 2063] payload_rows_per_dst=[4016, 4060, 4020, 4088, 4162, 4164, 4132, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4016, 4060, 4020, 4088, 4162, 4164, 4132, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2070, 2095, 2064, 2018, 2027, 2090, 2016, 2004] payload_rows_per_dst=[4140, 4190, 4128, 4036, 4054, 4180, 4032, 4008] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4140, 4190, 4128, 4036, 4054, 4180, 4032, 4008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28790 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2043, 2064, 2017, 2112, 1989, 2064, 2079] payload_rows_per_dst=[4032, 4086, 4128, 4034, 4224, 3978, 4128, 4158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4086, 4128, 4034, 4224, 3978, 4128, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114328 combine_payload_bytes=233930752 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234045080 ingress_bytes=2347859968 link_bytes=2347859968 token_bytes=467975832 weight_movement_bytes=2113929216 total_nvlink_bytes=2581905048 weight_to_token_ratio=4.517176 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314703578e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=237076480 dispatch_meta_bytes=115760 combine_payload_bytes=237076480 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=237192240 ingress_bytes=1646362624 link_bytes=1646362624 token_bytes=474268720 weight_movement_bytes=1409286144 total_nvlink_bytes=1883554864 weight_to_token_ratio=2.971493 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.218943469e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=232112128 dispatch_meta_bytes=113424 combine_payload_bytes=232112128 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=232225552 ingress_bytes=4459970560 link_bytes=4459970560 token_bytes=464337680 weight_movement_bytes=4227858432 total_nvlink_bytes=4692196112 weight_to_token_ratio=9.105138 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.497397345e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234586112 dispatch_meta_bytes=114544 combine_payload_bytes=234586112 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=234700656 ingress_bytes=939229184 link_bytes=939229184 token_bytes=469286768 weight_movement_bytes=704643072 total_nvlink_bytes=1173929840 weight_to_token_ratio=1.501519 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.259291377e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235225088 dispatch_meta_bytes=114856 combine_payload_bytes=235225088 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235339944 ingress_bytes=3758440448 link_bytes=3758440448 token_bytes=470565032 weight_movement_bytes=3523215360 total_nvlink_bytes=3993780392 weight_to_token_ratio=7.487202 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104569766e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2000, 2061, 2073, 1953, 2047, 2041, 2113, 2096] payload_rows_per_dst=[4000, 4122, 4146, 3906, 4094, 4082, 4226, 4192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4000, 4122, 4146, 3906, 4094, 4082, 4226, 4192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2086, 2003, 2086, 2035, 2060, 2031, 2012, 2071] payload_rows_per_dst=[4172, 4006, 4172, 4070, 4120, 4062, 4024, 4142] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4172, 4006, 4172, 4070, 4120, 4062, 4024, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235847680 dispatch_meta_bytes=115160 combine_payload_bytes=235847680 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235962840 ingress_bytes=2349776896 link_bytes=2349776896 token_bytes=471810520 weight_movement_bytes=2113929216 total_nvlink_bytes=2585739736 weight_to_token_ratio=4.480462 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315776977e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234668032 dispatch_meta_bytes=114584 combine_payload_bytes=234668032 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234782616 ingress_bytes=3053240320 link_bytes=3053240320 token_bytes=469450648 weight_movement_bytes=2818572288 total_nvlink_bytes=3288022936 weight_to_token_ratio=6.003980 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709687131e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=28434 dup_rows=96 dup_rows_remote=94 dedup_ratio=0.002930 router_tokens_per_expert=[2054, 2010, 2053, 2048, 2062, 2018, 2025, 2114] payload_rows_per_dst=[3616, 4056, 4046, 3882, 4284, 4132, 4418, 4238] dup_rows_per_dst=[0, 6, 0, 0, 40, 4, 44, 2] combine_rows_per_src=[3616, 4056, 4046, 3882, 4284, 4132, 4418, 4238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235208808 ingress_bytes=2349023232 link_bytes=2349023232 token_bytes=470302824 weight_movement_bytes=2113929216 total_nvlink_bytes=2584232040 weight_to_token_ratio=4.494826 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315354957e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2022, 1964, 2038, 2140, 2001, 2140, 2033, 2046] payload_rows_per_dst=[4044, 3928, 4076, 4280, 4002, 4280, 4066, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4044, 3928, 4076, 4280, 4002, 4280, 4066, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32678 payload_rows_remote=28480 dup_rows=90 dup_rows_remote=74 dedup_ratio=0.002747 router_tokens_per_expert=[2012, 1987, 2102, 2058, 2085, 2022, 2015, 2103] payload_rows_per_dst=[4198, 4282, 3836, 4170, 4284, 4044, 3902, 3962] dup_rows_per_dst=[16, 56, 0, 4, 14, 0, 0, 0] combine_rows_per_src=[4198, 4282, 3836, 4170, 4284, 4044, 3902, 3962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2059, 2004, 2079, 1986, 2079, 2028, 2075] payload_rows_per_dst=[4148, 4118, 4008, 4158, 3972, 4158, 4056, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4118, 4008, 4158, 3972, 4158, 4056, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2032, 1949, 2150, 2059, 2019, 2056, 2030, 2089] payload_rows_per_dst=[4064, 3898, 4300, 4118, 4038, 4112, 4060, 4178] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4064, 3898, 4300, 4118, 4038, 4112, 4060, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=232931328 dispatch_meta_bytes=114112 combine_payload_bytes=232931328 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233045440 ingress_bytes=2346860544 link_bytes=2346860544 token_bytes=465976768 weight_movement_bytes=2113929216 total_nvlink_bytes=2579905984 weight_to_token_ratio=4.536555 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314143942e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28808 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2102, 2006, 2073, 2061, 1980, 2047, 2056] payload_rows_per_dst=[4118, 4204, 4012, 4146, 4122, 3960, 4094, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4204, 4012, 4146, 4122, 3960, 4094, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235159632 ingress_bytes=2348974080 link_bytes=2348974080 token_bytes=470204496 weight_movement_bytes=2113929216 total_nvlink_bytes=2584133712 weight_to_token_ratio=4.495766 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315327434e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233308160 dispatch_meta_bytes=114216 combine_payload_bytes=233308160 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233422376 ingress_bytes=3756523520 link_bytes=3756523520 token_bytes=466730536 weight_movement_bytes=3523215360 total_nvlink_bytes=3989945896 weight_to_token_ratio=7.548714 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103496366e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235323552 ingress_bytes=1644494848 link_bytes=1644494848 token_bytes=470532256 weight_movement_bytes=1409286144 total_nvlink_bytes=1879818400 weight_to_token_ratio=2.995089 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208484703e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=114920 combine_payload_bytes=235356160 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235471080 ingress_bytes=1644642304 link_bytes=1644642304 token_bytes=470827240 weight_movement_bytes=1409286144 total_nvlink_bytes=1880113384 weight_to_token_ratio=2.993213 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209310395e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28552 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2108, 2082, 2023, 2073, 1981, 2012, 2046] payload_rows_per_dst=[4118, 4216, 4164, 4046, 4146, 3962, 4024, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4216, 4164, 4046, 4146, 3962, 4024, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28662 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2057, 2025, 2058, 2053, 2007, 2035, 2056, 2093] payload_rows_per_dst=[4114, 4050, 4116, 4106, 4014, 4070, 4112, 4186] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4114, 4050, 4116, 4106, 4014, 4070, 4112, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235995136 dispatch_meta_bytes=115232 combine_payload_bytes=235995136 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236110368 ingress_bytes=1645281280 link_bytes=1645281280 token_bytes=472105504 weight_movement_bytes=1409286144 total_nvlink_bytes=1881391648 weight_to_token_ratio=2.985108 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.212888394e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32678 payload_rows_remote=28700 dup_rows=90 dup_rows_remote=88 dedup_ratio=0.002747 router_tokens_per_expert=[2060, 2065, 2085, 2032, 2067, 2052, 2034, 1989] payload_rows_per_dst=[4000, 4130, 3874, 4300, 4176, 3950, 4270, 3978] dup_rows_per_dst=[0, 0, 0, 36, 6, 12, 34, 2] combine_rows_per_src=[4000, 4130, 3874, 4300, 4176, 3950, 4270, 3978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=233897984 dispatch_meta_bytes=114208 combine_payload_bytes=233897984 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234012192 ingress_bytes=1643184128 link_bytes=1643184128 token_bytes=467910176 weight_movement_bytes=1409286144 total_nvlink_bytes=1877196320 weight_to_token_ratio=3.011873 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201145218e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114648 combine_payload_bytes=234799104 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234913752 ingress_bytes=1644085248 link_bytes=1644085248 token_bytes=469712856 weight_movement_bytes=1409286144 total_nvlink_bytes=1878999000 weight_to_token_ratio=3.000314 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206191114e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28506 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 1988, 2131, 2005, 1996, 2054, 2082, 2097] payload_rows_per_dst=[4062, 3976, 4262, 4010, 3992, 4108, 4164, 4194] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 3976, 4262, 4010, 3992, 4108, 4164, 4194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2060, 2090, 1997, 2052, 2064, 2057, 2020] payload_rows_per_dst=[4088, 4120, 4180, 3994, 4104, 4128, 4114, 4040] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4120, 4180, 3994, 4104, 4128, 4114, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32650 payload_rows_remote=28368 dup_rows=118 dup_rows_remote=112 dedup_ratio=0.003601 router_tokens_per_expert=[2111, 2026, 2098, 2094, 1973, 2042, 2015, 2025] payload_rows_per_dst=[4282, 4104, 4298, 4240, 4148, 4048, 3562, 3968] dup_rows_per_dst=[6, 8, 30, 24, 26, 24, 0, 0] combine_rows_per_src=[4282, 4104, 4298, 4240, 4148, 4048, 3562, 3968] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2078, 2028, 1975, 2053, 2065, 2133, 2047, 2005] payload_rows_per_dst=[4156, 4056, 3950, 4106, 4130, 4266, 4094, 4010] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4156, 4056, 3950, 4106, 4130, 4266, 4094, 4010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=115152 combine_payload_bytes=235110400 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235225552 ingress_bytes=3053682688 link_bytes=3053682688 token_bytes=470335952 weight_movement_bytes=2818572288 total_nvlink_bytes=3288908240 weight_to_token_ratio=5.992679 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709934838e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2111, 2064, 1977, 2048, 2097, 2004, 2024] payload_rows_per_dst=[4118, 4222, 4128, 3954, 4096, 4194, 4008, 4048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4222, 4128, 3954, 4096, 4194, 4008, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233521152 dispatch_meta_bytes=114024 combine_payload_bytes=233521152 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=233635176 ingress_bytes=5166022656 link_bytes=5166022656 token_bytes=467156328 weight_movement_bytes=4932501504 total_nvlink_bytes=5399657832 weight_to_token_ratio=10.558567 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.892757047e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234848184 ingress_bytes=2348662784 link_bytes=2348662784 token_bytes=469581752 weight_movement_bytes=2113929216 total_nvlink_bytes=2583510968 weight_to_token_ratio=4.501728 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315153121e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234717048 ingress_bytes=1643888640 link_bytes=1643888640 token_bytes=469319544 weight_movement_bytes=1409286144 total_nvlink_bytes=1878605688 weight_to_token_ratio=3.002829 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205090191e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=232390656 dispatch_meta_bytes=113920 combine_payload_bytes=232390656 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232504576 ingress_bytes=3050962944 link_bytes=3050962944 token_bytes=464895232 weight_movement_bytes=2818572288 total_nvlink_bytes=3283467520 weight_to_token_ratio=6.062812 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708411895e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28608 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2070, 1995, 2080, 2015, 1958, 2133, 2103] payload_rows_per_dst=[4060, 4140, 3990, 4160, 4030, 3916, 4266, 4206] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4140, 3990, 4160, 4030, 3916, 4266, 4206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2017, 2036, 2038, 2022, 2075, 2052, 2090] payload_rows_per_dst=[4108, 4034, 4072, 4076, 4044, 4150, 4104, 4180] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4034, 4072, 4076, 4044, 4150, 4104, 4180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234078208 dispatch_meta_bytes=114296 combine_payload_bytes=234078208 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234192504 ingress_bytes=2348007424 link_bytes=2348007424 token_bytes=468270712 weight_movement_bytes=2113929216 total_nvlink_bytes=2582199928 weight_to_token_ratio=4.514331 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314786147e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32682 payload_rows_remote=28502 dup_rows=86 dup_rows_remote=86 dedup_ratio=0.002625 router_tokens_per_expert=[2067, 2095, 2008, 2110, 1998, 1940, 2076, 2090] payload_rows_per_dst=[3730, 4190, 4336, 4012, 4168, 3880, 4186, 4180] dup_rows_per_dst=[0, 0, 48, 0, 36, 0, 2, 0] combine_rows_per_src=[3730, 4190, 4336, 4012, 4168, 3880, 4186, 4180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234356736 dispatch_meta_bytes=114432 combine_payload_bytes=234356736 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234471168 ingress_bytes=3052929024 link_bytes=3052929024 token_bytes=468827904 weight_movement_bytes=2818572288 total_nvlink_bytes=3287400192 weight_to_token_ratio=6.011955 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709512818e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235503864 ingress_bytes=1644675072 link_bytes=1644675072 token_bytes=470892792 weight_movement_bytes=1409286144 total_nvlink_bytes=1880178936 weight_to_token_ratio=2.992796 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209493882e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2098, 2128, 2038, 1962, 2095, 1963, 2056, 2044] payload_rows_per_dst=[4196, 4256, 4076, 3924, 4190, 3926, 4112, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4196, 4256, 4076, 3924, 4190, 3926, 4112, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28514 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2108, 1953, 2002, 2070, 2002, 2089, 2127, 2033] payload_rows_per_dst=[4216, 3906, 4004, 4140, 4004, 4178, 4254, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4216, 3906, 4004, 4140, 4004, 4178, 4254, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28618 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1970, 2126, 2057, 2011, 2075, 2035, 2027, 2083] payload_rows_per_dst=[3940, 4252, 4114, 4022, 4150, 4070, 4054, 4166] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3940, 4252, 4114, 4022, 4150, 4070, 4054, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32694 payload_rows_remote=28318 dup_rows=74 dup_rows_remote=38 dedup_ratio=0.002258 router_tokens_per_expert=[2059, 2031, 2008, 2022, 2062, 2030, 2058, 2114] payload_rows_per_dst=[4376, 4124, 4004, 4070, 4174, 3966, 4116, 3864] dup_rows_per_dst=[36, 18, 0, 8, 12, 0, 0, 0] combine_rows_per_src=[4376, 4124, 4004, 4070, 4174, 3966, 4116, 3864] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233488384 dispatch_meta_bytes=114352 combine_payload_bytes=233488384 prefetch_bytes=0 reduce_bytes=2818572288 egress_bytes=233602736 ingress_bytes=3052060672 link_bytes=3052060672 token_bytes=467091120 weight_movement_bytes=2818572288 total_nvlink_bytes=3285663408 weight_to_token_ratio=6.034309 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709026577e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28442 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2009, 1990, 2051, 1993, 2163, 2088, 2024] payload_rows_per_dst=[4132, 4018, 3980, 4102, 3986, 4326, 4176, 4048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4018, 3980, 4102, 3986, 4326, 4176, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234438656 dispatch_meta_bytes=114472 combine_payload_bytes=234438656 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234553128 ingress_bytes=2348367872 link_bytes=2348367872 token_bytes=468991784 weight_movement_bytes=2113929216 total_nvlink_bytes=2582921000 weight_to_token_ratio=4.507391 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314987983e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233586688 dispatch_meta_bytes=114056 combine_payload_bytes=233586688 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=233700744 ingress_bytes=5166088192 link_bytes=5166088192 token_bytes=467287432 weight_movement_bytes=4932501504 total_nvlink_bytes=5399788936 weight_to_token_ratio=10.555605 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.892793744e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235159632 ingress_bytes=1644331008 link_bytes=1644331008 token_bytes=470204496 weight_movement_bytes=1409286144 total_nvlink_bytes=1879490640 weight_to_token_ratio=2.997177 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207567267e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28570 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2012, 2141, 2007, 2099, 2024, 2017, 1983, 2101] payload_rows_per_dst=[4024, 4282, 4014, 4198, 4048, 4034, 3966, 4202] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 4282, 4014, 4198, 4048, 4034, 3966, 4202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=231981056 dispatch_meta_bytes=113424 combine_payload_bytes=231981056 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232094480 ingress_bytes=1641267200 link_bytes=1641267200 token_bytes=464075536 weight_movement_bytes=1409286144 total_nvlink_bytes=1873361680 weight_to_token_ratio=3.036760 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.190411221e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28678 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1995, 2045, 2028, 2030, 2025, 2102, 2058, 2101] payload_rows_per_dst=[3990, 4090, 4056, 4060, 4050, 4204, 4116, 4202] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3990, 4090, 4056, 4060, 4050, 4204, 4116, 4202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=232996864 dispatch_meta_bytes=113768 combine_payload_bytes=232996864 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233110632 ingress_bytes=2346926080 link_bytes=2346926080 token_bytes=466107496 weight_movement_bytes=2113929216 total_nvlink_bytes=2580036712 weight_to_token_ratio=4.535283 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314180639e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234045440 dispatch_meta_bytes=114280 combine_payload_bytes=234045440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234159720 ingress_bytes=1643331584 link_bytes=1643331584 token_bytes=468205160 weight_movement_bytes=1409286144 total_nvlink_bytes=1877491304 weight_to_token_ratio=3.009976 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201970910e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32678 payload_rows_remote=28428 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.002747 router_tokens_per_expert=[2051, 1984, 2014, 1988, 2072, 2042, 2109, 2124] payload_rows_per_dst=[4102, 3720, 4110, 4038, 4074, 4134, 4250, 4250] dup_rows_per_dst=[0, 0, 54, 12, 0, 14, 10, 0] combine_rows_per_src=[4102, 3720, 4110, 4038, 4074, 4134, 4250, 4250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234930176 dispatch_meta_bytes=114712 combine_payload_bytes=234930176 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235044888 ingress_bytes=1644216320 link_bytes=1644216320 token_bytes=469975064 weight_movement_bytes=1409286144 total_nvlink_bytes=1879261208 weight_to_token_ratio=2.998640 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206925062e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2041, 2023, 2106, 2004, 2075, 2046, 2048] payload_rows_per_dst=[4082, 4082, 4046, 4212, 4008, 4150, 4092, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4082, 4046, 4212, 4008, 4150, 4092, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2007, 2044, 2095, 2079, 2050, 2033, 2004, 2072] payload_rows_per_dst=[4014, 4088, 4190, 4158, 4100, 4066, 4008, 4144] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4014, 4088, 4190, 4158, 4100, 4066, 4008, 4144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2067, 2038, 1963, 2071, 2076, 2045, 2060] payload_rows_per_dst=[4128, 4134, 4076, 3926, 4142, 4152, 4090, 4120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4134, 4076, 3926, 4142, 4152, 4090, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32742 payload_rows_remote=28586 dup_rows=26 dup_rows_remote=20 dedup_ratio=0.000793 router_tokens_per_expert=[2046, 2001, 2110, 2072, 2039, 2012, 2080, 2024] payload_rows_per_dst=[4156, 4122, 4280, 4058, 4034, 4024, 4124, 3944] dup_rows_per_dst=[6, 10, 10, 0, 0, 0, 0, 0] combine_rows_per_src=[4156, 4122, 4280, 4058, 4034, 4024, 4124, 3944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232882176 dispatch_meta_bytes=114072 combine_payload_bytes=232882176 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=232996248 ingress_bytes=4460740608 link_bytes=4460740608 token_bytes=465878424 weight_movement_bytes=4227858432 total_nvlink_bytes=4693736856 weight_to_token_ratio=9.075025 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.497828540e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2063, 2058, 2047, 2075, 1993, 2068, 2072, 2008] payload_rows_per_dst=[4126, 4116, 4094, 4150, 3986, 4136, 4144, 4016] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4126, 4116, 4094, 4150, 3986, 4136, 4144, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235601920 dispatch_meta_bytes=115040 combine_payload_bytes=235601920 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235716960 ingress_bytes=2349531136 link_bytes=2349531136 token_bytes=471318880 weight_movement_bytes=2113929216 total_nvlink_bytes=2585248096 weight_to_token_ratio=4.485136 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315639362e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235601920 dispatch_meta_bytes=115040 combine_payload_bytes=235601920 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235716960 ingress_bytes=1644888064 link_bytes=1644888064 token_bytes=471318880 weight_movement_bytes=1409286144 total_nvlink_bytes=1880605024 weight_to_token_ratio=2.990091 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210686548e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28534 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2062, 2052, 1983, 2117, 2110, 1935, 2088, 2037] payload_rows_per_dst=[4124, 4104, 3966, 4234, 4220, 3870, 4176, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4124, 4104, 3966, 4234, 4220, 3870, 4176, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235159632 ingress_bytes=3758260224 link_bytes=3758260224 token_bytes=470204496 weight_movement_bytes=3523215360 total_nvlink_bytes=3993419856 weight_to_token_ratio=7.492943 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104468848e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234176512 dispatch_meta_bytes=114424 combine_payload_bytes=234176512 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234290936 ingress_bytes=3757391872 link_bytes=3757391872 token_bytes=468467448 weight_movement_bytes=3523215360 total_nvlink_bytes=3991682808 weight_to_token_ratio=7.520726 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103982607e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 2, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2011, 2017, 2056, 2093, 2041, 2098, 2065, 2003] payload_rows_per_dst=[4022, 4034, 4112, 4186, 4082, 4196, 4130, 4006] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4022, 4034, 4112, 4186, 4082, 4196, 4130, 4006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234667872 ingress_bytes=3757768704 link_bytes=3757768704 token_bytes=469221216 weight_movement_bytes=3523215360 total_nvlink_bytes=3992436576 weight_to_token_ratio=7.508645 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104193617e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233750528 dispatch_meta_bytes=114136 combine_payload_bytes=233750528 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233864664 ingress_bytes=3756965888 link_bytes=3756965888 token_bytes=467615192 weight_movement_bytes=3523215360 total_nvlink_bytes=3990830552 weight_to_token_ratio=7.534433 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103744074e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=28586 dup_rows=48 dup_rows_remote=40 dedup_ratio=0.001465 router_tokens_per_expert=[2064, 2017, 2073, 2090, 1970, 2047, 2097, 2026] payload_rows_per_dst=[3896, 3966, 4146, 4068, 4040, 4274, 4196, 4134] dup_rows_per_dst=[0, 0, 0, 0, 12, 28, 0, 8] combine_rows_per_src=[3896, 3966, 4146, 4068, 4040, 4274, 4196, 4134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28664 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2014, 2105, 1961, 2077, 2052, 2083, 2067, 2025] payload_rows_per_dst=[4028, 4210, 3922, 4154, 4104, 4166, 4134, 4050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4028, 4210, 3922, 4154, 4104, 4166, 4134, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235503864 ingress_bytes=2349318144 link_bytes=2349318144 token_bytes=470892792 weight_movement_bytes=2113929216 total_nvlink_bytes=2584822008 weight_to_token_ratio=4.489194 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315520095e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28630 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2101, 2038, 2065, 1967, 2050, 2060, 2069, 2034] payload_rows_per_dst=[4202, 4076, 4130, 3934, 4100, 4120, 4138, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4202, 4076, 4130, 3934, 4100, 4120, 4138, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 2066, 2026, 2042, 2013, 2007, 2062, 2121] payload_rows_per_dst=[4094, 4132, 4052, 4084, 4026, 4014, 4124, 4242] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 4132, 4052, 4084, 4026, 4014, 4124, 4242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32652 payload_rows_remote=28418 dup_rows=116 dup_rows_remote=100 dedup_ratio=0.003540 router_tokens_per_expert=[2082, 1999, 2063, 2010, 2111, 2059, 2066, 1994] payload_rows_per_dst=[4234, 4272, 4176, 4206, 4078, 3842, 4026, 3818] dup_rows_per_dst=[16, 44, 8, 34, 14, 0, 0, 0] combine_rows_per_src=[4234, 4272, 4176, 4206, 4078, 3842, 4026, 3818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234176512 dispatch_meta_bytes=114504 combine_payload_bytes=234176512 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234291016 ingress_bytes=2348105728 link_bytes=2348105728 token_bytes=468467528 weight_movement_bytes=2113929216 total_nvlink_bytes=2582396744 weight_to_token_ratio=4.512435 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314841193e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2064, 2045, 2071, 2083, 2087, 1958, 2038] payload_rows_per_dst=[4076, 4128, 4090, 4142, 4166, 4174, 3916, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4128, 4090, 4142, 4166, 4174, 3916, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234815488 dispatch_meta_bytes=114656 combine_payload_bytes=234815488 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234930144 ingress_bytes=2348744704 link_bytes=2348744704 token_bytes=469745632 weight_movement_bytes=2113929216 total_nvlink_bytes=2583674848 weight_to_token_ratio=4.500157 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315198993e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28634 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2085, 2046, 2067, 2006, 2057, 2030, 2049] payload_rows_per_dst=[4088, 4170, 4092, 4134, 4012, 4114, 4060, 4098] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4170, 4092, 4134, 4012, 4114, 4060, 4098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114520 combine_payload_bytes=234536960 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234651480 ingress_bytes=2348466176 link_bytes=2348466176 token_bytes=469188440 weight_movement_bytes=2113929216 total_nvlink_bytes=2583117656 weight_to_token_ratio=4.505501 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315043029e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235356336 ingress_bytes=1644527616 link_bytes=1644527616 token_bytes=470597808 weight_movement_bytes=1409286144 total_nvlink_bytes=1879883952 weight_to_token_ratio=2.994672 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208668190e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=232800256 dispatch_meta_bytes=114072 combine_payload_bytes=232800256 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232914328 ingress_bytes=2346729472 link_bytes=2346729472 token_bytes=465714584 weight_movement_bytes=2113929216 total_nvlink_bytes=2579643800 weight_to_token_ratio=4.539109 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314070547e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28620 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2074, 2024, 2018, 2081, 2057, 2001, 2063] payload_rows_per_dst=[4132, 4148, 4048, 4036, 4162, 4114, 4002, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4148, 4048, 4036, 4162, 4114, 4002, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234242048 dispatch_meta_bytes=114376 combine_payload_bytes=234242048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234356424 ingress_bytes=1643528192 link_bytes=1643528192 token_bytes=468598472 weight_movement_bytes=1409286144 total_nvlink_bytes=1877884616 weight_to_token_ratio=3.007449 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203071833e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234569728 dispatch_meta_bytes=114536 combine_payload_bytes=234569728 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234684264 ingress_bytes=1643855872 link_bytes=1643855872 token_bytes=469253992 weight_movement_bytes=1409286144 total_nvlink_bytes=1878540136 weight_to_token_ratio=3.003248 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204906704e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32746 payload_rows_remote=28580 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000671 router_tokens_per_expert=[2029, 1994, 2056, 2045, 2092, 1997, 2088, 2083] payload_rows_per_dst=[3974, 3960, 4182, 4090, 4184, 4014, 4176, 4166] dup_rows_per_dst=[0, 0, 14, 0, 0, 8, 0, 0] combine_rows_per_src=[3974, 3960, 4182, 4090, 4184, 4014, 4176, 4166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2068, 2062, 1993, 2097, 2028, 2098, 2022] payload_rows_per_dst=[4032, 4136, 4124, 3986, 4194, 4056, 4196, 4044] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4136, 4124, 3986, 4194, 4056, 4196, 4044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234455040 dispatch_meta_bytes=114480 combine_payload_bytes=234455040 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234569520 ingress_bytes=2348384256 link_bytes=2348384256 token_bytes=469024560 weight_movement_bytes=2113929216 total_nvlink_bytes=2582953776 weight_to_token_ratio=4.507076 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314997157e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28690 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2052, 2024, 1969, 2046, 2084, 2078, 2039, 2092] payload_rows_per_dst=[4104, 4048, 3938, 4092, 4168, 4156, 4078, 4184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4104, 4048, 3938, 4092, 4168, 4156, 4078, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2101, 2055, 2068, 1959, 2001, 2078, 2094, 2028] payload_rows_per_dst=[4202, 4110, 4136, 3918, 4002, 4156, 4188, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4202, 4110, 4136, 3918, 4002, 4156, 4188, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234127360 dispatch_meta_bytes=114408 combine_payload_bytes=234127360 prefetch_bytes=0 reduce_bytes=3523215360 egress_bytes=234241768 ingress_bytes=3757342720 link_bytes=3757342720 token_bytes=468369128 weight_movement_bytes=3523215360 total_nvlink_bytes=3991584488 weight_to_token_ratio=7.522305 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103955084e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32682 payload_rows_remote=28332 dup_rows=86 dup_rows_remote=78 dedup_ratio=0.002625 router_tokens_per_expert=[2141, 1970, 2059, 2000, 2059, 2130, 2008, 2017] payload_rows_per_dst=[4350, 4026, 4126, 4212, 4032, 3998, 4014, 3924] dup_rows_per_dst=[8, 22, 6, 50, 0, 0, 0, 0] combine_rows_per_src=[4350, 4026, 4126, 4212, 4032, 3998, 4014, 3924] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2046, 1969, 2121, 2049, 2022, 2062, 2019, 2096] payload_rows_per_dst=[4092, 3938, 4242, 4098, 4044, 4124, 4038, 4192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4092, 3938, 4242, 4098, 4044, 4124, 4038, 4192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234078208 dispatch_meta_bytes=114296 combine_payload_bytes=234078208 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234192504 ingress_bytes=3757293568 link_bytes=3757293568 token_bytes=468270712 weight_movement_bytes=3523215360 total_nvlink_bytes=3991486072 weight_to_token_ratio=7.523886 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103927561e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28636 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2073, 2109, 1990, 2066, 2019, 2078, 2023, 2026] payload_rows_per_dst=[4146, 4218, 3980, 4132, 4038, 4156, 4046, 4052] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 4218, 3980, 4132, 4038, 4156, 4046, 4052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235028480 dispatch_meta_bytes=114760 combine_payload_bytes=235028480 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235143240 ingress_bytes=3053600768 link_bytes=3053600768 token_bytes=470171720 weight_movement_bytes=2818572288 total_nvlink_bytes=3288744008 weight_to_token_ratio=5.994772 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709888967e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234667872 ingress_bytes=1643839488 link_bytes=1643839488 token_bytes=469221216 weight_movement_bytes=1409286144 total_nvlink_bytes=1878507360 weight_to_token_ratio=3.003458 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204814960e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2015, 2008, 2029, 2021, 2081, 2094, 2034, 2102] payload_rows_per_dst=[4030, 4016, 4058, 4042, 4162, 4188, 4068, 4204] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4030, 4016, 4058, 4042, 4162, 4188, 4068, 4204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234766224 ingress_bytes=4462510080 link_bytes=4462510080 token_bytes=469417872 weight_movement_bytes=4227858432 total_nvlink_bytes=4697276304 weight_to_token_ratio=9.006599 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498819370e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232095744 dispatch_meta_bytes=113640 combine_payload_bytes=232095744 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=232209384 ingress_bytes=3755311104 link_bytes=3755311104 token_bytes=464305128 weight_movement_bytes=3523215360 total_nvlink_bytes=3987520488 weight_to_token_ratio=7.588147 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.102817464e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234586112 dispatch_meta_bytes=114544 combine_payload_bytes=234586112 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234700656 ingress_bytes=1643872256 link_bytes=1643872256 token_bytes=469286768 weight_movement_bytes=1409286144 total_nvlink_bytes=1878572912 weight_to_token_ratio=3.003038 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204998448e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7528122784 prefetch_bytes=19730006016 reduce_bytes=14092861440 weight_movement_bytes=33822867456 egress_bytes=3764980128 ingress_bytes=37586010112 link_bytes=37586010112 total_nvlink_bytes=41350990240 weight_to_token_ratio=4.492869 prefetch_share_of_ingress=0.524930 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=20 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.315412297e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32662 payload_rows_remote=28374 dup_rows=106 dup_rows_remote=80 dedup_ratio=0.003235 router_tokens_per_expert=[2168, 1982, 2032, 1967, 2000, 2106, 2113, 2016] payload_rows_per_dst=[3664, 3924, 3970, 4272, 3834, 4278, 4432, 4288] dup_rows_per_dst=[0, 0, 0, 48, 0, 10, 22, 26] combine_rows_per_src=[3664, 3924, 3970, 4272, 3834, 4278, 4432, 4288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235651392 ingress_bytes=2349465600 link_bytes=2349465600 token_bytes=471187776 weight_movement_bytes=2113929216 total_nvlink_bytes=2585116992 weight_to_token_ratio=4.486384 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315602664e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7467622088 prefetch_bytes=16206790656 reduce_bytes=31708938240 weight_movement_bytes=47915728896 egress_bytes=3734724296 ingress_bytes=51648626688 link_bytes=51648626688 total_nvlink_bytes=55383350984 weight_to_token_ratio=6.416464 prefetch_share_of_ingress=0.313789 dedup_ratio=0.002228 prefetch_slots=46 unused_prefetch_slots=18 slot_utilization=0.718750 reduce_slots=45 remote_demand=23 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.807567189e-04 prefetch_bytes_per_gemm_flop=5.671953914e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7516552856 prefetch_bytes=18320719872 reduce_bytes=16206790656 weight_movement_bytes=34527510528 egress_bytes=3759193752 ingress_bytes=38284869632 link_bytes=38284869632 total_nvlink_bytes=42044063384 weight_to_token_ratio=4.593530 prefetch_share_of_ingress=0.478537 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=23 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.339870557e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1994, 2101, 2006, 2015, 2163, 2018, 2048, 2039] payload_rows_per_dst=[3988, 4202, 4012, 4030, 4326, 4036, 4096, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3988, 4202, 4012, 4030, 4326, 4036, 4096, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28570 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2015, 2072, 2099, 2021, 2131, 2013, 1968, 2065] payload_rows_per_dst=[4030, 4144, 4198, 4042, 4262, 4026, 3936, 4130] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4030, 4144, 4198, 4042, 4262, 4026, 3936, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232439808 dispatch_meta_bytes=113816 combine_payload_bytes=232439808 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232553624 ingress_bytes=2346369024 link_bytes=2346369024 token_bytes=464993432 weight_movement_bytes=2113929216 total_nvlink_bytes=2578922648 weight_to_token_ratio=4.546149 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313868711e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2195, 1990, 2087, 1977, 1998, 2014, 2041, 2082] payload_rows_per_dst=[4390, 3980, 4174, 3954, 3996, 4028, 4082, 4164] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4390, 3980, 4174, 3954, 3996, 4028, 4082, 4164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234995712 ingress_bytes=2348810240 link_bytes=2348810240 token_bytes=469876736 weight_movement_bytes=2113929216 total_nvlink_bytes=2583805952 weight_to_token_ratio=4.498902 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315235690e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234045440 dispatch_meta_bytes=114280 combine_payload_bytes=234045440 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234159720 ingress_bytes=3052617728 link_bytes=3052617728 token_bytes=468205160 weight_movement_bytes=2818572288 total_nvlink_bytes=3286777448 weight_to_token_ratio=6.019951 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709338505e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7473391224 prefetch_bytes=17616076800 reduce_bytes=25367150592 weight_movement_bytes=42983227392 egress_bytes=3737609848 ingress_bytes=46719008768 link_bytes=46719008768 total_nvlink_bytes=50456618616 weight_to_token_ratio=5.751502 prefetch_share_of_ingress=0.377064 dedup_ratio=0.002491 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=36 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.635043423e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2058, 2089, 2021, 2043, 2090, 1972, 2031] payload_rows_per_dst=[4160, 4116, 4178, 4042, 4086, 4180, 3944, 4062] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4116, 4178, 4042, 4086, 4180, 3944, 4062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114960 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235553040 ingress_bytes=2349367296 link_bytes=2349367296 token_bytes=470991120 weight_movement_bytes=2113929216 total_nvlink_bytes=2584920336 weight_to_token_ratio=4.488257 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315547618e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7515471248 prefetch_bytes=19025362944 reduce_bytes=16206790656 weight_movement_bytes=35232153600 egress_bytes=3758652816 ingress_bytes=38988972032 link_bytes=38988972032 total_nvlink_bytes=42747624848 weight_to_token_ratio=4.687950 prefetch_share_of_ingress=0.487968 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=23 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.364512304e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7521469256 prefetch_bytes=21139292160 reduce_bytes=16206790656 weight_movement_bytes=37346082816 egress_bytes=3761652552 ingress_bytes=41105899520 link_bytes=41105899520 total_nvlink_bytes=44867552072 weight_to_token_ratio=4.965264 prefetch_share_of_ingress=0.514264 dedup_ratio=0.000000 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=23 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016098 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.438599243e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234831792 ingress_bytes=2348646400 link_bytes=2348646400 token_bytes=469548976 weight_movement_bytes=2113929216 total_nvlink_bytes=2583478192 weight_to_token_ratio=4.502042 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315143947e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7515897336 prefetch_bytes=21843935232 reduce_bytes=14092861440 weight_movement_bytes=35936796672 egress_bytes=3758865912 ingress_bytes=39693828096 link_bytes=39693828096 total_nvlink_bytes=43452694008 weight_to_token_ratio=4.781438 prefetch_share_of_ingress=0.550311 dedup_ratio=0.000000 prefetch_slots=62 unused_prefetch_slots=2 slot_utilization=0.968750 reduce_slots=20 remote_demand=31 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.015625 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.389180427e-04 prefetch_bytes_per_gemm_flop=7.644807449e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7521928120 prefetch_bytes=19025362944 reduce_bytes=19025362944 weight_movement_bytes=38050725888 egress_bytes=3761882040 ingress_bytes=41810771968 link_bytes=41810771968 total_nvlink_bytes=45572654008 weight_to_token_ratio=5.058640 prefetch_share_of_ingress=0.455035 dedup_ratio=0.000000 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=27 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.463267940e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none - [2026-09-16 21:35:55] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 579.4 | throughput per GPU (TFLOP/s/GPU): 575.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.570642E+00 | loss scale: 1.0 | grad norm: 4.278 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32704 payload_rows_remote=28422 dup_rows=64 dup_rows_remote=50 dedup_ratio=0.001953 router_tokens_per_expert=[2073, 2032, 1989, 2014, 2029, 2108, 2095, 2044] payload_rows_per_dst=[4282, 4230, 3810, 4230, 4058, 4112, 4120, 3862] dup_rows_per_dst=[14, 18, 0, 32, 0, 0, 0, 0] combine_rows_per_src=[4282, 4230, 3810, 4230, 4058, 4112, 4120, 3862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 1945, 2131, 2027, 2055, 2062, 2066, 2034] payload_rows_per_dst=[4128, 3890, 4262, 4054, 4110, 4124, 4132, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 3890, 4262, 4054, 4110, 4124, 4132, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28770 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2014, 2073, 2070, 2086, 1999, 2047, 2032, 2063] payload_rows_per_dst=[4028, 4146, 4140, 4172, 3998, 4094, 4064, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4028, 4146, 4140, 4172, 3998, 4094, 4064, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=232833024 dispatch_meta_bytes=113888 combine_payload_bytes=232833024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232946912 ingress_bytes=1642119168 link_bytes=1642119168 token_bytes=465779936 weight_movement_bytes=1409286144 total_nvlink_bytes=1875066080 weight_to_token_ratio=3.025648 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195181887e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234766224 ingress_bytes=3053223936 link_bytes=3053223936 token_bytes=469417872 weight_movement_bytes=2818572288 total_nvlink_bytes=3287990160 weight_to_token_ratio=6.004399 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709677956e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=28412 dup_rows=76 dup_rows_remote=48 dedup_ratio=0.002319 router_tokens_per_expert=[2022, 2011, 2132, 2079, 2028, 2054, 2009, 2049] payload_rows_per_dst=[4044, 4022, 4198, 3902, 3902, 4092, 4252, 4280] dup_rows_per_dst=[0, 0, 0, 0, 0, 34, 14, 28] combine_rows_per_src=[4044, 4022, 4198, 3902, 3902, 4092, 4252, 4280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28724 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2078, 2050, 1991, 2039, 2079, 2045, 2022, 2080] payload_rows_per_dst=[4156, 4100, 3982, 4078, 4158, 4090, 4044, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4156, 4100, 3982, 4078, 4158, 4090, 4044, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235683840 dispatch_meta_bytes=115080 combine_payload_bytes=235683840 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235798920 ingress_bytes=3054256128 link_bytes=3054256128 token_bytes=471482760 weight_movement_bytes=2818572288 total_nvlink_bytes=3290055048 weight_to_token_ratio=5.978103 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710255941e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1978, 2023, 2055, 2068, 2138, 2043, 2033, 2046] payload_rows_per_dst=[3956, 4046, 4110, 4136, 4276, 4086, 4066, 4092] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3956, 4046, 4110, 4136, 4276, 4086, 4066, 4092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28710 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2090, 2079, 2029, 2052, 2025, 2043, 2015] payload_rows_per_dst=[4102, 4180, 4158, 4058, 4104, 4050, 4086, 4030] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4180, 4158, 4058, 4104, 4050, 4086, 4030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=28490 dup_rows=76 dup_rows_remote=68 dedup_ratio=0.002319 router_tokens_per_expert=[2091, 2034, 2005, 1967, 2063, 2059, 2044, 2121] payload_rows_per_dst=[4202, 4278, 3790, 4030, 4090, 4044, 4088, 4170] dup_rows_per_dst=[8, 34, 0, 14, 20, 0, 0, 0] combine_rows_per_src=[4202, 4278, 3790, 4030, 4090, 4044, 4088, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=232751104 dispatch_meta_bytes=113840 combine_payload_bytes=232751104 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232864944 ingress_bytes=2346680320 link_bytes=2346680320 token_bytes=465616048 weight_movement_bytes=2113929216 total_nvlink_bytes=2579545264 weight_to_token_ratio=4.540069 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314043024e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2096, 2106, 2058, 2018, 2120, 2003, 1958, 2025] payload_rows_per_dst=[4192, 4212, 4116, 4036, 4240, 4006, 3916, 4050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4192, 4212, 4116, 4036, 4240, 4006, 3916, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235307008 dispatch_meta_bytes=114896 combine_payload_bytes=235307008 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235421904 ingress_bytes=3053879296 link_bytes=3053879296 token_bytes=470728912 weight_movement_bytes=2818572288 total_nvlink_bytes=3289301200 weight_to_token_ratio=5.987676 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710044931e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234880968 ingress_bytes=4462624768 link_bytes=4462624768 token_bytes=469647304 weight_movement_bytes=4227858432 total_nvlink_bytes=4697505736 weight_to_token_ratio=9.002199 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498883591e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=233390080 dispatch_meta_bytes=114232 combine_payload_bytes=233390080 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233504312 ingress_bytes=3051962368 link_bytes=3051962368 token_bytes=466894392 weight_movement_bytes=2818572288 total_nvlink_bytes=3285466680 weight_to_token_ratio=6.036852 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708971531e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235192320 dispatch_meta_bytes=114840 combine_payload_bytes=235192320 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235307160 ingress_bytes=2349121536 link_bytes=2349121536 token_bytes=470499480 weight_movement_bytes=2113929216 total_nvlink_bytes=2584428696 weight_to_token_ratio=4.492947 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315410003e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2081, 1993, 2041, 2071, 2057, 2011, 2092, 2038] payload_rows_per_dst=[4162, 3986, 4082, 4142, 4114, 4022, 4184, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4162, 3986, 4082, 4142, 4114, 4022, 4184, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235618304 dispatch_meta_bytes=115048 combine_payload_bytes=235618304 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235733352 ingress_bytes=3054190592 link_bytes=3054190592 token_bytes=471351656 weight_movement_bytes=2818572288 total_nvlink_bytes=3289923944 weight_to_token_ratio=5.979765 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710219243e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2068, 2054, 2053, 2081, 2043, 1995, 2061] payload_rows_per_dst=[4058, 4136, 4108, 4106, 4162, 4086, 3990, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4136, 4108, 4106, 4162, 4086, 3990, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234848184 ingress_bytes=4462592000 link_bytes=4462592000 token_bytes=469581752 weight_movement_bytes=4227858432 total_nvlink_bytes=4697440184 weight_to_token_ratio=9.003456 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498865242e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28422 dup_rows=80 dup_rows_remote=58 dedup_ratio=0.002441 router_tokens_per_expert=[2018, 2005, 1985, 2050, 2111, 2066, 2092, 2057] payload_rows_per_dst=[3682, 4010, 3982, 4386, 4026, 4140, 4196, 4266] dup_rows_per_dst=[0, 0, 0, 50, 0, 0, 8, 22] combine_rows_per_src=[3682, 4010, 3982, 4386, 4026, 4140, 4196, 4266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32748 payload_rows_remote=28630 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000610 router_tokens_per_expert=[2050, 2068, 2064, 2028, 2117, 2052, 2032, 1973] payload_rows_per_dst=[4118, 4118, 4128, 4138, 4146, 4140, 4064, 3896] dup_rows_per_dst=[0, 0, 0, 6, 0, 14, 0, 0] combine_rows_per_src=[4118, 4118, 4128, 4138, 4146, 4140, 4064, 3896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2076, 2068, 1977, 1993, 2087, 2044, 2023, 2116] payload_rows_per_dst=[4152, 4136, 3954, 3986, 4174, 4088, 4046, 4232] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4152, 4136, 3954, 3986, 4174, 4088, 4046, 4232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2012, 2084, 2020, 2075, 2079, 2032, 2028] payload_rows_per_dst=[4108, 4024, 4168, 4040, 4150, 4158, 4064, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4024, 4168, 4040, 4150, 4158, 4064, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234667872 ingress_bytes=1643839488 link_bytes=1643839488 token_bytes=469221216 weight_movement_bytes=1409286144 total_nvlink_bytes=1878507360 weight_to_token_ratio=3.003458 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204814960e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28602 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2036, 2095, 2083, 2022, 1999, 2022, 2046, 2081] payload_rows_per_dst=[4072, 4190, 4166, 4044, 3998, 4044, 4092, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4072, 4190, 4166, 4044, 3998, 4044, 4092, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=232833024 dispatch_meta_bytes=113920 combine_payload_bytes=232833024 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232946944 ingress_bytes=2346762240 link_bytes=2346762240 token_bytes=465779968 weight_movement_bytes=2113929216 total_nvlink_bytes=2579709184 weight_to_token_ratio=4.538472 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314088896e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28748 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2088, 2009, 2103, 2010, 2024, 2106, 2044, 2000] payload_rows_per_dst=[4176, 4018, 4206, 4020, 4048, 4212, 4088, 4000] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4176, 4018, 4206, 4020, 4048, 4212, 4088, 4000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28786 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2022, 2056, 2166, 1991, 2007, 1993, 2100] payload_rows_per_dst=[4098, 4044, 4112, 4332, 3982, 4014, 3986, 4200] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4044, 4112, 4332, 3982, 4014, 3986, 4200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114600 combine_payload_bytes=234536960 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234651560 ingress_bytes=2348466176 link_bytes=2348466176 token_bytes=469188520 weight_movement_bytes=2113929216 total_nvlink_bytes=2583117736 weight_to_token_ratio=4.505501 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315043029e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235061280 ingress_bytes=1644232704 link_bytes=1644232704 token_bytes=470007840 weight_movement_bytes=1409286144 total_nvlink_bytes=1879293984 weight_to_token_ratio=2.998431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207016806e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235257984 ingress_bytes=2349072384 link_bytes=2349072384 token_bytes=470401152 weight_movement_bytes=2113929216 total_nvlink_bytes=2584330368 weight_to_token_ratio=4.493886 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315382480e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234307584 dispatch_meta_bytes=114408 combine_payload_bytes=234307584 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234421992 ingress_bytes=3052879872 link_bytes=3052879872 token_bytes=468729576 weight_movement_bytes=2818572288 total_nvlink_bytes=3287301864 weight_to_token_ratio=6.013216 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709485295e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235814912 dispatch_meta_bytes=115144 combine_payload_bytes=235814912 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235930056 ingress_bytes=3054387200 link_bytes=3054387200 token_bytes=471744968 weight_movement_bytes=2818572288 total_nvlink_bytes=3290317256 weight_to_token_ratio=5.974780 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710329336e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235503616 dispatch_meta_bytes=114992 combine_payload_bytes=235503616 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235618608 ingress_bytes=1644789760 link_bytes=1644789760 token_bytes=471122224 weight_movement_bytes=1409286144 total_nvlink_bytes=1880408368 weight_to_token_ratio=2.991339 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210136087e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2083, 2008, 2129, 2003, 2041, 2071, 2066, 1983] payload_rows_per_dst=[4166, 4016, 4258, 4006, 4082, 4142, 4132, 3966] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4166, 4016, 4258, 4006, 4082, 4142, 4132, 3966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32650 payload_rows_remote=28374 dup_rows=118 dup_rows_remote=94 dedup_ratio=0.003601 router_tokens_per_expert=[2073, 1964, 2059, 1976, 2037, 2118, 2085, 2072] payload_rows_per_dst=[4276, 4020, 4358, 4080, 3974, 4078, 3978, 3886] dup_rows_per_dst=[24, 18, 54, 20, 2, 0, 0, 0] combine_rows_per_src=[4276, 4020, 4358, 4080, 3974, 4078, 3978, 3886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28558 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2039, 2103, 2012, 2007, 2105, 2011, 2033] payload_rows_per_dst=[4148, 4078, 4206, 4024, 4014, 4210, 4022, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4078, 4206, 4024, 4014, 4210, 4022, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32644 payload_rows_remote=28540 dup_rows=124 dup_rows_remote=114 dedup_ratio=0.003784 router_tokens_per_expert=[2021, 2057, 2094, 2064, 2030, 2078, 2014, 2026] payload_rows_per_dst=[4024, 3850, 4326, 3816, 3932, 4314, 4278, 4104] dup_rows_per_dst=[0, 0, 34, 0, 8, 18, 54, 10] combine_rows_per_src=[4024, 3850, 4326, 3816, 3932, 4314, 4278, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28746 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2082, 2080, 2064, 2067, 1989, 2011, 2035] payload_rows_per_dst=[4112, 4164, 4160, 4128, 4134, 3978, 4022, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4164, 4160, 4128, 4134, 3978, 4022, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2161, 1977, 2113, 2040, 2032, 1994, 2009] payload_rows_per_dst=[4116, 4322, 3954, 4226, 4080, 4064, 3988, 4018] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4322, 3954, 4226, 4080, 4064, 3988, 4018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235651392 ingress_bytes=1644822528 link_bytes=1644822528 token_bytes=471187776 weight_movement_bytes=1409286144 total_nvlink_bytes=1880473920 weight_to_token_ratio=2.990923 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210319574e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2012, 2074, 2043, 2066, 2077, 1985, 2123, 2004] payload_rows_per_dst=[4024, 4148, 4086, 4132, 4154, 3970, 4246, 4008] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4024, 4148, 4086, 4132, 4154, 3970, 4246, 4008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=233947136 dispatch_meta_bytes=114232 combine_payload_bytes=233947136 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234061368 ingress_bytes=3052519424 link_bytes=3052519424 token_bytes=468008504 weight_movement_bytes=2818572288 total_nvlink_bytes=3286580792 weight_to_token_ratio=6.022481 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709283459e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=232439808 dispatch_meta_bytes=113872 combine_payload_bytes=232439808 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232553680 ingress_bytes=3051012096 link_bytes=3051012096 token_bytes=464993488 weight_movement_bytes=2818572288 total_nvlink_bytes=3283565776 weight_to_token_ratio=6.061531 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708439418e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233799680 dispatch_meta_bytes=114616 combine_payload_bytes=233799680 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233914296 ingress_bytes=2347728896 link_bytes=2347728896 token_bytes=467713976 weight_movement_bytes=2113929216 total_nvlink_bytes=2581643192 weight_to_token_ratio=4.519705 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314630183e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28702 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2050, 2046, 2047, 2033, 2070, 2054, 2067, 2017] payload_rows_per_dst=[4100, 4092, 4094, 4066, 4140, 4108, 4134, 4034] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4100, 4092, 4094, 4066, 4140, 4108, 4134, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235487232 dispatch_meta_bytes=114984 combine_payload_bytes=235487232 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235602216 ingress_bytes=1644773376 link_bytes=1644773376 token_bytes=471089448 weight_movement_bytes=1409286144 total_nvlink_bytes=1880375592 weight_to_token_ratio=2.991547 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210044343e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=236044288 dispatch_meta_bytes=115256 combine_payload_bytes=236044288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236159544 ingress_bytes=1645330432 link_bytes=1645330432 token_bytes=472203832 weight_movement_bytes=1409286144 total_nvlink_bytes=1881489976 weight_to_token_ratio=2.984487 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213163624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234520344 ingress_bytes=3757621248 link_bytes=3757621248 token_bytes=468926232 weight_movement_bytes=3523215360 total_nvlink_bytes=3992141592 weight_to_token_ratio=7.513368 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104111048e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235126784 dispatch_meta_bytes=114808 combine_payload_bytes=235126784 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235241592 ingress_bytes=3053699072 link_bytes=3053699072 token_bytes=470368376 weight_movement_bytes=2818572288 total_nvlink_bytes=3288940664 weight_to_token_ratio=5.992266 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709944013e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28604 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 1994, 2055, 2065, 2032, 2082, 2105, 2016] payload_rows_per_dst=[4070, 3988, 4110, 4130, 4064, 4164, 4210, 4032] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 3988, 4110, 4130, 4064, 4164, 4210, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28544 dup_rows=80 dup_rows_remote=66 dedup_ratio=0.002441 router_tokens_per_expert=[2049, 2076, 2004, 2053, 2044, 2050, 2032, 2076] payload_rows_per_dst=[4144, 4166, 4040, 4138, 4150, 3996, 4118, 3936] dup_rows_per_dst=[14, 6, 6, 14, 6, 0, 34, 0] combine_rows_per_src=[4144, 4166, 4040, 4138, 4150, 3996, 4118, 3936] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32604 payload_rows_remote=28216 dup_rows=164 dup_rows_remote=112 dedup_ratio=0.005005 router_tokens_per_expert=[2007, 2110, 2075, 1980, 2088, 2026, 2000, 2098] payload_rows_per_dst=[3794, 4082, 3736, 4318, 4064, 4102, 4120, 4388] dup_rows_per_dst=[0, 0, 0, 56, 6, 32, 18, 52] combine_rows_per_src=[3794, 4082, 3736, 4318, 4064, 4102, 4120, 4388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28726 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2021, 2003, 2035, 2086, 2015, 2098, 2102] payload_rows_per_dst=[4048, 4042, 4006, 4070, 4172, 4030, 4196, 4204] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4042, 4006, 4070, 4172, 4030, 4196, 4204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2087, 2062, 2015, 2051, 2111, 2025, 2032, 2001] payload_rows_per_dst=[4174, 4124, 4030, 4102, 4222, 4050, 4064, 4002] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4174, 4124, 4030, 4102, 4222, 4050, 4064, 4002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28630 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2069, 2057, 2069, 1974, 2063, 2031, 2040, 2081] payload_rows_per_dst=[4138, 4114, 4138, 3948, 4126, 4062, 4080, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4138, 4114, 4138, 3948, 4126, 4062, 4080, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28692 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2022, 2067, 2029, 2050, 2038, 1992, 2058, 2128] payload_rows_per_dst=[4044, 4134, 4058, 4100, 4076, 3984, 4116, 4256] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4044, 4134, 4058, 4100, 4076, 3984, 4116, 4256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234323968 dispatch_meta_bytes=114416 combine_payload_bytes=234323968 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234438384 ingress_bytes=2348253184 link_bytes=2348253184 token_bytes=468762352 weight_movement_bytes=2113929216 total_nvlink_bytes=2582691568 weight_to_token_ratio=4.509597 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314923762e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233832448 dispatch_meta_bytes=114440 combine_payload_bytes=233832448 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233946888 ingress_bytes=3052404736 link_bytes=3052404736 token_bytes=467779336 weight_movement_bytes=2818572288 total_nvlink_bytes=3286351624 weight_to_token_ratio=6.025431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709219239e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=231145472 dispatch_meta_bytes=113312 combine_payload_bytes=231145472 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=231258784 ingress_bytes=3049717760 link_bytes=3049717760 token_bytes=462404256 weight_movement_bytes=2818572288 total_nvlink_bytes=3280976544 weight_to_token_ratio=6.095472 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.707714644e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2035, 2051, 1986, 2035, 2093, 2096, 2010, 2078] payload_rows_per_dst=[4070, 4102, 3972, 4070, 4186, 4192, 4020, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4070, 4102, 3972, 4070, 4186, 4192, 4020, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=235323392 dispatch_meta_bytes=114904 combine_payload_bytes=235323392 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=235438296 ingress_bytes=4463181824 link_bytes=4463181824 token_bytes=470761688 weight_movement_bytes=4227858432 total_nvlink_bytes=4698620120 weight_to_token_ratio=8.980889 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499195519e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235257984 ingress_bytes=2349072384 link_bytes=2349072384 token_bytes=470401152 weight_movement_bytes=2113929216 total_nvlink_bytes=2584330368 weight_to_token_ratio=4.493886 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315382480e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114520 combine_payload_bytes=234536960 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234651480 ingress_bytes=2348466176 link_bytes=2348466176 token_bytes=469188440 weight_movement_bytes=2113929216 total_nvlink_bytes=2583117656 weight_to_token_ratio=4.505501 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315043029e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235044864 dispatch_meta_bytes=114768 combine_payload_bytes=235044864 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235159632 ingress_bytes=2348974080 link_bytes=2348974080 token_bytes=470204496 weight_movement_bytes=2113929216 total_nvlink_bytes=2584133712 weight_to_token_ratio=4.495766 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315327434e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235208808 ingress_bytes=1644380160 link_bytes=1644380160 token_bytes=470302824 weight_movement_bytes=1409286144 total_nvlink_bytes=1879588968 weight_to_token_ratio=2.996550 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207842498e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2065, 1999, 2079, 2097, 2073, 2047, 1969] payload_rows_per_dst=[4110, 4130, 3998, 4158, 4194, 4146, 4094, 3938] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4130, 3998, 4158, 4194, 4146, 4094, 3938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32656 payload_rows_remote=28498 dup_rows=112 dup_rows_remote=84 dedup_ratio=0.003418 router_tokens_per_expert=[2114, 2048, 2043, 2050, 2062, 2007, 2056, 2004] payload_rows_per_dst=[3990, 4096, 4010, 4100, 3960, 4332, 4010, 4158] dup_rows_per_dst=[0, 0, 0, 0, 24, 60, 0, 28] combine_rows_per_src=[3990, 4096, 4010, 4100, 3960, 4332, 4010, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=28572 dup_rows=48 dup_rows_remote=40 dedup_ratio=0.001465 router_tokens_per_expert=[2037, 1979, 2014, 2127, 2044, 2063, 2060, 2060] payload_rows_per_dst=[4148, 4154, 4032, 4170, 4082, 3986, 4120, 4028] dup_rows_per_dst=[8, 34, 6, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4154, 4032, 4170, 4082, 3986, 4120, 4028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28654 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2041, 2017, 2085, 2006, 2080, 2027, 2057, 2071] payload_rows_per_dst=[4082, 4034, 4170, 4012, 4160, 4054, 4114, 4142] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4082, 4034, 4170, 4012, 4160, 4054, 4114, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28556 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2052, 2022, 2106, 2042, 2046, 2035, 1978, 2103] payload_rows_per_dst=[4104, 4044, 4212, 4084, 4092, 4070, 3956, 4206] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4104, 4044, 4212, 4084, 4092, 4070, 3956, 4206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28686 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 2041, 2094, 2009, 2010, 2036, 2110, 2024] payload_rows_per_dst=[4120, 4082, 4188, 4018, 4020, 4072, 4220, 4048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 4082, 4188, 4018, 4020, 4072, 4220, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114488 combine_payload_bytes=234471424 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234585912 ingress_bytes=2348400640 link_bytes=2348400640 token_bytes=469057336 weight_movement_bytes=2113929216 total_nvlink_bytes=2582986552 weight_to_token_ratio=4.506761 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315006331e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234061824 dispatch_meta_bytes=114448 combine_payload_bytes=234061824 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234176272 ingress_bytes=3052634112 link_bytes=3052634112 token_bytes=468238096 weight_movement_bytes=2818572288 total_nvlink_bytes=3286810384 weight_to_token_ratio=6.019528 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709347680e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28758 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2027, 2107, 2044, 2108, 2005, 2005, 2008, 2080] payload_rows_per_dst=[4054, 4214, 4088, 4216, 4010, 4010, 4016, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4054, 4214, 4088, 4216, 4010, 4010, 4016, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233455616 dispatch_meta_bytes=114328 combine_payload_bytes=233455616 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233569944 ingress_bytes=2347384832 link_bytes=2347384832 token_bytes=467025560 weight_movement_bytes=2113929216 total_nvlink_bytes=2580954776 weight_to_token_ratio=4.526367 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314437521e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2105, 2028, 2062, 2014, 2062, 2100, 2046, 1967] payload_rows_per_dst=[4210, 4056, 4124, 4028, 4124, 4200, 4092, 3934] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4210, 4056, 4124, 4028, 4124, 4200, 4092, 3934] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234733568 dispatch_meta_bytes=114616 combine_payload_bytes=234733568 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234848184 ingress_bytes=3757948928 link_bytes=3757948928 token_bytes=469581752 weight_movement_bytes=3523215360 total_nvlink_bytes=3992797112 weight_to_token_ratio=7.502880 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104294535e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114224 combine_payload_bytes=233930752 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234044976 ingress_bytes=2347859968 link_bytes=2347859968 token_bytes=467975728 weight_movement_bytes=2113929216 total_nvlink_bytes=2581904944 weight_to_token_ratio=4.517177 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314703578e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234995712 dispatch_meta_bytes=114744 combine_payload_bytes=234995712 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235110456 ingress_bytes=2348924928 link_bytes=2348924928 token_bytes=470106168 weight_movement_bytes=2113929216 total_nvlink_bytes=2584035384 weight_to_token_ratio=4.496706 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315299911e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114960 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235553040 ingress_bytes=3054010368 link_bytes=3054010368 token_bytes=470991120 weight_movement_bytes=2818572288 total_nvlink_bytes=3289563408 weight_to_token_ratio=5.984343 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710118325e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235585536 dispatch_meta_bytes=115032 combine_payload_bytes=235585536 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235700568 ingress_bytes=3054157824 link_bytes=3054157824 token_bytes=471286104 weight_movement_bytes=2818572288 total_nvlink_bytes=3289858392 weight_to_token_ratio=5.980597 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710200895e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28726 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2094, 1998, 2023, 2022, 2061, 2021, 2097, 2068] payload_rows_per_dst=[4188, 3996, 4046, 4044, 4122, 4042, 4194, 4136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4188, 3996, 4046, 4044, 4122, 4042, 4194, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32710 payload_rows_remote=28500 dup_rows=58 dup_rows_remote=56 dedup_ratio=0.001770 router_tokens_per_expert=[2098, 2146, 2108, 1965, 2024, 1985, 2031, 2027] payload_rows_per_dst=[4210, 4282, 4418, 4040, 4048, 3970, 3774, 3968] dup_rows_per_dst=[2, 0, 38, 18, 0, 0, 0, 0] combine_rows_per_src=[4210, 4282, 4418, 4040, 4048, 3970, 3774, 3968] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32636 payload_rows_remote=28246 dup_rows=132 dup_rows_remote=98 dedup_ratio=0.004028 router_tokens_per_expert=[2112, 2012, 2012, 2010, 2055, 2073, 2009, 2101] payload_rows_per_dst=[4014, 4024, 4166, 3594, 3830, 4332, 4286, 4390] dup_rows_per_dst=[0, 0, 22, 0, 0, 34, 42, 34] combine_rows_per_src=[4014, 4024, 4166, 3594, 3830, 4332, 4286, 4390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28734 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2078, 2039, 2032, 2135, 2095, 1999, 2017, 1989] payload_rows_per_dst=[4156, 4078, 4064, 4270, 4190, 3998, 4034, 3978] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4156, 4078, 4064, 4270, 4190, 3998, 4034, 3978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2029, 2068, 2103, 2027, 2030, 2083, 2015] payload_rows_per_dst=[4058, 4058, 4136, 4206, 4054, 4060, 4166, 4030] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4058, 4136, 4206, 4054, 4060, 4166, 4030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235323392 dispatch_meta_bytes=114904 combine_payload_bytes=235323392 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235438296 ingress_bytes=1644609536 link_bytes=1644609536 token_bytes=470761688 weight_movement_bytes=1409286144 total_nvlink_bytes=1880047832 weight_to_token_ratio=2.993630 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209126908e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28736 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 1999, 2066, 2016, 2105, 2042, 1980, 2110] payload_rows_per_dst=[4132, 3998, 4132, 4032, 4210, 4084, 3960, 4220] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 3998, 4132, 4032, 4210, 4084, 3960, 4220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114224 combine_payload_bytes=233472000 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233586224 ingress_bytes=4461330432 link_bytes=4461330432 token_bytes=467058224 weight_movement_bytes=4227858432 total_nvlink_bytes=4694916656 weight_to_token_ratio=9.052101 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498158817e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28524 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2122, 2019, 2081, 2056, 1954, 2059, 2038] payload_rows_per_dst=[4110, 4244, 4038, 4162, 4112, 3908, 4118, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4244, 4038, 4162, 4112, 3908, 4118, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28624 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2095, 1969, 2078, 2079, 2072, 2021, 2028, 2042] payload_rows_per_dst=[4190, 3938, 4156, 4158, 4144, 4042, 4056, 4084] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4190, 3938, 4156, 4158, 4144, 4042, 4056, 4084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=231391232 dispatch_meta_bytes=113376 combine_payload_bytes=231391232 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=231504608 ingress_bytes=2345320448 link_bytes=2345320448 token_bytes=462895840 weight_movement_bytes=2113929216 total_nvlink_bytes=2576825056 weight_to_token_ratio=4.566749 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313281552e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235388928 dispatch_meta_bytes=114936 combine_payload_bytes=235388928 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235503864 ingress_bytes=1644675072 link_bytes=1644675072 token_bytes=470892792 weight_movement_bytes=1409286144 total_nvlink_bytes=1880178936 weight_to_token_ratio=2.992796 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209493882e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234667872 ingress_bytes=2348482560 link_bytes=2348482560 token_bytes=469221216 weight_movement_bytes=2113929216 total_nvlink_bytes=2583150432 weight_to_token_ratio=4.505187 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315052203e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235405312 dispatch_meta_bytes=114944 combine_payload_bytes=235405312 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=235520256 ingress_bytes=5167906816 link_bytes=5167906816 token_bytes=470925568 weight_movement_bytes=4932501504 total_nvlink_bytes=5403427072 weight_to_token_ratio=10.474058 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893812098e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233668608 dispatch_meta_bytes=114096 combine_payload_bytes=233668608 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233782704 ingress_bytes=2347597824 link_bytes=2347597824 token_bytes=467451312 weight_movement_bytes=2113929216 total_nvlink_bytes=2581380528 weight_to_token_ratio=4.522245 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314556788e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234487808 dispatch_meta_bytes=114496 combine_payload_bytes=234487808 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234602304 ingress_bytes=1643773952 link_bytes=1643773952 token_bytes=469090112 weight_movement_bytes=1409286144 total_nvlink_bytes=1878376256 weight_to_token_ratio=3.004297 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204447986e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28674 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2075, 2081, 2038, 2060, 1950, 2047, 2015, 2118] payload_rows_per_dst=[4150, 4162, 4076, 4120, 3900, 4094, 4030, 4236] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4150, 4162, 4076, 4120, 3900, 4094, 4030, 4236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32698 payload_rows_remote=28476 dup_rows=70 dup_rows_remote=28 dedup_ratio=0.002136 router_tokens_per_expert=[2023, 2042, 2019, 2112, 2005, 2110, 2066, 2007] payload_rows_per_dst=[4222, 3882, 4148, 4218, 4066, 4134, 4082, 3946] dup_rows_per_dst=[42, 0, 16, 0, 12, 0, 0, 0] combine_rows_per_src=[4222, 3882, 4148, 4218, 4066, 4134, 4082, 3946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32712 payload_rows_remote=28616 dup_rows=56 dup_rows_remote=54 dedup_ratio=0.001709 router_tokens_per_expert=[2032, 2083, 2039, 2015, 2045, 2058, 2064, 2048] payload_rows_per_dst=[3776, 4160, 3960, 4030, 4132, 4328, 4230, 4096] dup_rows_per_dst=[0, 2, 0, 0, 12, 24, 16, 2] combine_rows_per_src=[3776, 4160, 3960, 4030, 4132, 4328, 4230, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28690 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2028, 2051, 2056, 2040, 2018, 2124, 2039, 2028] payload_rows_per_dst=[4056, 4102, 4112, 4080, 4036, 4248, 4078, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4056, 4102, 4112, 4080, 4036, 4248, 4078, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234897408 dispatch_meta_bytes=114696 combine_payload_bytes=234897408 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235012104 ingress_bytes=3053469696 link_bytes=3053469696 token_bytes=469909512 weight_movement_bytes=2818572288 total_nvlink_bytes=3288481800 weight_to_token_ratio=5.998117 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709815572e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28794 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2089, 2017, 1987, 2072, 2046, 1995, 2083, 2095] payload_rows_per_dst=[4178, 4034, 3974, 4144, 4092, 3990, 4166, 4190] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4178, 4034, 3974, 4144, 4092, 3990, 4166, 4190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28698 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2009, 2067, 2006, 2035, 2038, 2077, 2067, 2085] payload_rows_per_dst=[4018, 4134, 4012, 4070, 4076, 4154, 4134, 4170] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4018, 4134, 4012, 4070, 4076, 4154, 4134, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233275392 dispatch_meta_bytes=114016 combine_payload_bytes=233275392 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233389408 ingress_bytes=3051847680 link_bytes=3051847680 token_bytes=466664800 weight_movement_bytes=2818572288 total_nvlink_bytes=3285237088 weight_to_token_ratio=6.039822 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708907310e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28690 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 2039, 2062, 2061, 2099, 2059, 2025, 1995] payload_rows_per_dst=[4088, 4078, 4124, 4122, 4198, 4118, 4050, 3990] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 4078, 4124, 4122, 4198, 4118, 4050, 3990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234422272 dispatch_meta_bytes=114680 combine_payload_bytes=234422272 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234536952 ingress_bytes=3052994560 link_bytes=3052994560 token_bytes=468959224 weight_movement_bytes=2818572288 total_nvlink_bytes=3287531512 weight_to_token_ratio=6.010272 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709549515e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28706 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2036, 2036, 2068, 2031, 2070, 2025, 2054] payload_rows_per_dst=[4128, 4072, 4072, 4136, 4062, 4140, 4050, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4072, 4072, 4136, 4062, 4140, 4050, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235028480 dispatch_meta_bytes=114760 combine_payload_bytes=235028480 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235143240 ingress_bytes=2348957696 link_bytes=2348957696 token_bytes=470171720 weight_movement_bytes=2113929216 total_nvlink_bytes=2584100936 weight_to_token_ratio=4.496079 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315318259e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235094016 dispatch_meta_bytes=114792 combine_payload_bytes=235094016 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235208808 ingress_bytes=1644380160 link_bytes=1644380160 token_bytes=470302824 weight_movement_bytes=1409286144 total_nvlink_bytes=1879588968 weight_to_token_ratio=2.996550 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207842498e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235880448 dispatch_meta_bytes=115176 combine_payload_bytes=235880448 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235995624 ingress_bytes=2349809664 link_bytes=2349809664 token_bytes=471876072 weight_movement_bytes=2113929216 total_nvlink_bytes=2585805288 weight_to_token_ratio=4.479840 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315795326e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235028480 dispatch_meta_bytes=114760 combine_payload_bytes=235028480 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235143240 ingress_bytes=1644314624 link_bytes=1644314624 token_bytes=470171720 weight_movement_bytes=1409286144 total_nvlink_bytes=1879457864 weight_to_token_ratio=2.997386 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207475524e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235159552 dispatch_meta_bytes=114824 combine_payload_bytes=235159552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235274376 ingress_bytes=1644445696 link_bytes=1644445696 token_bytes=470433928 weight_movement_bytes=1409286144 total_nvlink_bytes=1879720072 weight_to_token_ratio=2.995715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208209472e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2051, 2083, 2039, 2015, 2020, 2074, 2065] payload_rows_per_dst=[4074, 4102, 4166, 4078, 4030, 4040, 4148, 4130] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4102, 4166, 4078, 4030, 4040, 4148, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32652 payload_rows_remote=28354 dup_rows=116 dup_rows_remote=52 dedup_ratio=0.003540 router_tokens_per_expert=[1942, 2120, 2046, 2038, 2064, 2047, 2083, 2044] payload_rows_per_dst=[4298, 3888, 4284, 4242, 4118, 3978, 4276, 3568] dup_rows_per_dst=[64, 0, 28, 8, 0, 0, 16, 0] combine_rows_per_src=[4298, 3888, 4284, 4242, 4118, 3978, 4276, 3568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32668 payload_rows_remote=28506 dup_rows=100 dup_rows_remote=72 dedup_ratio=0.003052 router_tokens_per_expert=[1998, 2135, 2066, 2050, 2038, 2008, 2073, 2016] payload_rows_per_dst=[3848, 3966, 4132, 3922, 4164, 4170, 4304, 4162] dup_rows_per_dst=[0, 0, 0, 0, 16, 32, 24, 28] combine_rows_per_src=[3848, 3966, 4132, 3922, 4164, 4170, 4304, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235454688 ingress_bytes=3758555136 link_bytes=3758555136 token_bytes=470794464 weight_movement_bytes=3523215360 total_nvlink_bytes=3994009824 weight_to_token_ratio=7.483553 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104633986e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28624 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2017, 2036, 2056, 2063, 2056, 2072, 2047] payload_rows_per_dst=[4074, 4034, 4072, 4112, 4126, 4112, 4144, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4034, 4072, 4112, 4126, 4112, 4144, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2033, 2126, 2115, 2028, 2071, 1992, 1970, 2049] payload_rows_per_dst=[4066, 4252, 4230, 4056, 4142, 3984, 3940, 4098] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4066, 4252, 4230, 4056, 4142, 3984, 3940, 4098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28820 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2031, 2128, 1974, 2057, 2078, 2050, 2019, 2047] payload_rows_per_dst=[4062, 4256, 3948, 4114, 4156, 4100, 4038, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4062, 4256, 3948, 4114, 4156, 4100, 4038, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28616 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2014, 2039, 2084, 2006, 2076, 2021, 2028, 2116] payload_rows_per_dst=[4028, 4078, 4168, 4012, 4152, 4042, 4056, 4232] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4028, 4078, 4168, 4012, 4152, 4042, 4056, 4232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2075, 2060, 2026, 2092, 2053, 1985, 2060, 2033] payload_rows_per_dst=[4150, 4120, 4052, 4184, 4106, 3970, 4120, 4066] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4150, 4120, 4052, 4184, 4106, 3970, 4120, 4066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=232275968 dispatch_meta_bytes=113624 combine_payload_bytes=232275968 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232389592 ingress_bytes=1641562112 link_bytes=1641562112 token_bytes=464665560 weight_movement_bytes=1409286144 total_nvlink_bytes=1873951704 weight_to_token_ratio=3.032904 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.192062605e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233521152 dispatch_meta_bytes=114312 combine_payload_bytes=233521152 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233635464 ingress_bytes=2347450368 link_bytes=2347450368 token_bytes=467156616 weight_movement_bytes=2113929216 total_nvlink_bytes=2581085832 weight_to_token_ratio=4.525097 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314474219e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234487808 dispatch_meta_bytes=114496 combine_payload_bytes=234487808 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234602304 ingress_bytes=2348417024 link_bytes=2348417024 token_bytes=469090112 weight_movement_bytes=2113929216 total_nvlink_bytes=2583019328 weight_to_token_ratio=4.506446 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315015506e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235323552 ingress_bytes=3053780992 link_bytes=3053780992 token_bytes=470532256 weight_movement_bytes=2818572288 total_nvlink_bytes=3289104544 weight_to_token_ratio=5.990179 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709989884e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=236093440 dispatch_meta_bytes=115280 combine_payload_bytes=236093440 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236208720 ingress_bytes=1645379584 link_bytes=1645379584 token_bytes=472302160 weight_movement_bytes=1409286144 total_nvlink_bytes=1881588304 weight_to_token_ratio=2.983866 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213438855e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234422272 dispatch_meta_bytes=114464 combine_payload_bytes=234422272 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234536736 ingress_bytes=2348351488 link_bytes=2348351488 token_bytes=468959008 weight_movement_bytes=2113929216 total_nvlink_bytes=2582888224 weight_to_token_ratio=4.507706 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314978808e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234799008 ingress_bytes=3053256704 link_bytes=3053256704 token_bytes=469483424 weight_movement_bytes=2818572288 total_nvlink_bytes=3288055712 weight_to_token_ratio=6.003561 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709696305e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2055, 2075, 2059, 1992, 2037, 2057, 2051] payload_rows_per_dst=[4116, 4110, 4150, 4118, 3984, 4074, 4114, 4102] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4110, 4150, 4118, 3984, 4074, 4114, 4102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32726 payload_rows_remote=28418 dup_rows=42 dup_rows_remote=22 dedup_ratio=0.001282 router_tokens_per_expert=[2086, 2076, 2043, 1977, 1961, 2098, 2048, 2095] payload_rows_per_dst=[4172, 4152, 3840, 4080, 3936, 4128, 4110, 4308] dup_rows_per_dst=[0, 0, 0, 20, 2, 0, 0, 20] combine_rows_per_src=[4172, 4152, 3840, 4080, 3936, 4128, 4110, 4308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28606 dup_rows=80 dup_rows_remote=74 dedup_ratio=0.002441 router_tokens_per_expert=[2013, 2107, 2037, 2052, 2043, 2059, 2031, 2042] payload_rows_per_dst=[4082, 4278, 4050, 4398, 4092, 4072, 3978, 3738] dup_rows_per_dst=[6, 20, 0, 52, 2, 0, 0, 0] combine_rows_per_src=[4082, 4278, 4050, 4398, 4092, 4072, 3978, 3738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235176024 ingress_bytes=3053633536 link_bytes=3053633536 token_bytes=470237272 weight_movement_bytes=2818572288 total_nvlink_bytes=3288809560 weight_to_token_ratio=5.993936 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709907315e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28874 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2062, 2049, 2060, 2095, 2035, 1947, 2078] payload_rows_per_dst=[4116, 4124, 4098, 4120, 4190, 4070, 3894, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4124, 4098, 4120, 4190, 4070, 3894, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28744 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2022, 2052, 2055, 2012, 2089, 2093, 2042, 2019] payload_rows_per_dst=[4044, 4104, 4110, 4024, 4178, 4186, 4084, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4044, 4104, 4110, 4024, 4178, 4186, 4084, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28598 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 2049, 2012, 2033, 2085, 1962, 2092, 2091] payload_rows_per_dst=[4120, 4098, 4024, 4066, 4170, 3924, 4184, 4182] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 4098, 4024, 4066, 4170, 3924, 4184, 4182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28748 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1984, 2063, 2010, 2079, 2043, 2110, 2048, 2047] payload_rows_per_dst=[3968, 4126, 4020, 4158, 4086, 4220, 4096, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3968, 4126, 4020, 4158, 4086, 4220, 4096, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28624 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2088, 2072, 2037, 2045, 2062, 2001, 2078, 2001] payload_rows_per_dst=[4176, 4144, 4074, 4090, 4124, 4002, 4156, 4002] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4176, 4144, 4074, 4090, 4124, 4002, 4156, 4002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=232800256 dispatch_meta_bytes=113760 combine_payload_bytes=232800256 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=232914016 ingress_bytes=3756015616 link_bytes=3756015616 token_bytes=465714272 weight_movement_bytes=3523215360 total_nvlink_bytes=3988929632 weight_to_token_ratio=7.565187 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103211961e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114720 combine_payload_bytes=234340352 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234455072 ingress_bytes=3052912640 link_bytes=3052912640 token_bytes=468795424 weight_movement_bytes=2818572288 total_nvlink_bytes=3287367712 weight_to_token_ratio=6.012372 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709503644e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=236535808 dispatch_meta_bytes=115496 combine_payload_bytes=236535808 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236651304 ingress_bytes=1645821952 link_bytes=1645821952 token_bytes=473187112 weight_movement_bytes=1409286144 total_nvlink_bytes=1882473256 weight_to_token_ratio=2.978285 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.215915931e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235470848 dispatch_meta_bytes=114976 combine_payload_bytes=235470848 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235585824 ingress_bytes=1644756992 link_bytes=1644756992 token_bytes=471056672 weight_movement_bytes=1409286144 total_nvlink_bytes=1880342816 weight_to_token_ratio=2.991755 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209952600e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234274816 dispatch_meta_bytes=114392 combine_payload_bytes=234274816 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234389208 ingress_bytes=1643560960 link_bytes=1643560960 token_bytes=468664024 weight_movement_bytes=1409286144 total_nvlink_bytes=1877950168 weight_to_token_ratio=3.007029 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203255320e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235503616 dispatch_meta_bytes=114992 combine_payload_bytes=235503616 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=235618608 ingress_bytes=3758718976 link_bytes=3758718976 token_bytes=471122224 weight_movement_bytes=3523215360 total_nvlink_bytes=3994337584 weight_to_token_ratio=7.478347 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104725730e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234487808 dispatch_meta_bytes=114496 combine_payload_bytes=234487808 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234602304 ingress_bytes=4462346240 link_bytes=4462346240 token_bytes=469090112 weight_movement_bytes=4227858432 total_nvlink_bytes=4696948544 weight_to_token_ratio=9.012892 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498727627e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28834 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2026, 2072, 2037, 2084, 2023, 1967, 2030, 2145] payload_rows_per_dst=[4052, 4144, 4074, 4168, 4046, 3934, 4060, 4290] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4052, 4144, 4074, 4168, 4046, 3934, 4060, 4290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32636 payload_rows_remote=28234 dup_rows=132 dup_rows_remote=58 dedup_ratio=0.004028 router_tokens_per_expert=[2045, 1996, 1984, 2026, 2046, 2173, 2052, 2062] payload_rows_per_dst=[3870, 3712, 3858, 3934, 4220, 4522, 4118, 4402] dup_rows_per_dst=[0, 0, 0, 0, 14, 42, 2, 74] combine_rows_per_src=[3870, 3712, 3858, 3934, 4220, 4522, 4118, 4402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=236208128 dispatch_meta_bytes=115336 combine_payload_bytes=236208128 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236323464 ingress_bytes=2350137344 link_bytes=2350137344 token_bytes=472531592 weight_movement_bytes=2113929216 total_nvlink_bytes=2586460808 weight_to_token_ratio=4.473625 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315978813e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32652 payload_rows_remote=28414 dup_rows=116 dup_rows_remote=112 dedup_ratio=0.003540 router_tokens_per_expert=[2101, 2029, 2007, 2051, 2051, 2002, 2065, 2078] payload_rows_per_dst=[4238, 4334, 4224, 4006, 4128, 4140, 3924, 3658] dup_rows_per_dst=[4, 52, 32, 0, 2, 26, 0, 0] combine_rows_per_src=[4238, 4334, 4224, 4006, 4128, 4140, 3924, 3658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2097, 2116, 1986, 1990, 2050, 2077, 1994] payload_rows_per_dst=[4148, 4194, 4232, 3972, 3980, 4100, 4154, 3988] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4194, 4232, 3972, 3980, 4100, 4154, 3988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2039, 2048, 2092, 2006, 2051, 2017, 2069, 2062] payload_rows_per_dst=[4078, 4096, 4184, 4012, 4102, 4034, 4138, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4078, 4096, 4184, 4012, 4102, 4034, 4138, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2053, 1997, 2093, 2032, 2076, 2064, 2035] payload_rows_per_dst=[4068, 4106, 3994, 4186, 4064, 4152, 4128, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4106, 3994, 4186, 4064, 4152, 4128, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=231292928 dispatch_meta_bytes=113168 combine_payload_bytes=231292928 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231406096 ingress_bytes=2345222144 link_bytes=2345222144 token_bytes=462699024 weight_movement_bytes=2113929216 total_nvlink_bytes=2576628240 weight_to_token_ratio=4.568692 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313226506e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28636 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2096, 2090, 2066, 2084, 2082, 1974, 1981, 2011] payload_rows_per_dst=[4192, 4180, 4132, 4168, 4164, 3948, 3962, 4022] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4192, 4180, 4132, 4168, 4164, 3948, 3962, 4022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28574 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2002, 2097, 2027, 2013, 2038, 2052, 2044, 2111] payload_rows_per_dst=[4004, 4194, 4054, 4026, 4076, 4104, 4088, 4222] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4004, 4194, 4054, 4026, 4076, 4104, 4088, 4222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=232767488 dispatch_meta_bytes=114104 combine_payload_bytes=232767488 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232881592 ingress_bytes=2346696704 link_bytes=2346696704 token_bytes=465649080 weight_movement_bytes=2113929216 total_nvlink_bytes=2579578296 weight_to_token_ratio=4.539747 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314052198e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234520344 ingress_bytes=2348335104 link_bytes=2348335104 token_bytes=468926232 weight_movement_bytes=2113929216 total_nvlink_bytes=2582855448 weight_to_token_ratio=4.508021 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314969634e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235257984 ingress_bytes=1644429312 link_bytes=1644429312 token_bytes=470401152 weight_movement_bytes=1409286144 total_nvlink_bytes=1879687296 weight_to_token_ratio=2.995924 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208117729e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235684176 ingress_bytes=2349498368 link_bytes=2349498368 token_bytes=471253328 weight_movement_bytes=2113929216 total_nvlink_bytes=2585182544 weight_to_token_ratio=4.485760 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315621013e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28666 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2078, 2051, 2065, 2076, 2020, 2051, 2018, 2025] payload_rows_per_dst=[4156, 4102, 4130, 4152, 4040, 4102, 4036, 4050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4156, 4102, 4130, 4152, 4040, 4102, 4036, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=234586112 dispatch_meta_bytes=114544 combine_payload_bytes=234586112 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234700656 ingress_bytes=3757801472 link_bytes=3757801472 token_bytes=469286768 weight_movement_bytes=3523215360 total_nvlink_bytes=3992502128 weight_to_token_ratio=7.507596 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104211966e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234078208 dispatch_meta_bytes=114296 combine_payload_bytes=234078208 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234192504 ingress_bytes=3052650496 link_bytes=3052650496 token_bytes=468270712 weight_movement_bytes=2818572288 total_nvlink_bytes=3286843000 weight_to_token_ratio=6.019109 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709356854e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32738 payload_rows_remote=28486 dup_rows=30 dup_rows_remote=12 dedup_ratio=0.000916 router_tokens_per_expert=[2026, 2026, 2024, 2078, 2085, 2007, 2065, 2073] payload_rows_per_dst=[3986, 4050, 4048, 4152, 4224, 3890, 4136, 4252] dup_rows_per_dst=[0, 0, 0, 0, 12, 0, 0, 18] combine_rows_per_src=[3986, 4050, 4048, 4152, 4224, 3890, 4136, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234831872 dispatch_meta_bytes=114664 combine_payload_bytes=234831872 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234946536 ingress_bytes=2348761088 link_bytes=2348761088 token_bytes=469778408 weight_movement_bytes=2113929216 total_nvlink_bytes=2583707624 weight_to_token_ratio=4.499843 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315208167e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=28512 dup_rows=52 dup_rows_remote=32 dedup_ratio=0.001587 router_tokens_per_expert=[2019, 2003, 2012, 2079, 2098, 2015, 2101, 2057] payload_rows_per_dst=[4204, 3984, 4026, 4222, 3978, 4030, 4158, 4114] dup_rows_per_dst=[20, 20, 0, 12, 0, 0, 0, 0] combine_rows_per_src=[4204, 3984, 4026, 4222, 3978, 4030, 4158, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28670 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2088, 2011, 2052, 2024, 2051, 2075, 2049, 2034] payload_rows_per_dst=[4176, 4022, 4104, 4048, 4102, 4150, 4098, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4176, 4022, 4104, 4048, 4102, 4150, 4098, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28810 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2036, 2083, 2069, 1979, 2025, 2040, 2081] payload_rows_per_dst=[4142, 4072, 4166, 4138, 3958, 4050, 4080, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4072, 4166, 4138, 3958, 4050, 4080, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 1999, 2034, 2062, 2056, 2089, 2065, 2023] payload_rows_per_dst=[4112, 3998, 4068, 4124, 4112, 4178, 4130, 4046] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 3998, 4068, 4124, 4112, 4178, 4130, 4046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233357312 dispatch_meta_bytes=113992 combine_payload_bytes=233357312 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233471304 ingress_bytes=3051929600 link_bytes=3051929600 token_bytes=466828616 weight_movement_bytes=2818572288 total_nvlink_bytes=3285400904 weight_to_token_ratio=6.037702 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708953182e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2078, 2040, 2069, 2024, 2020, 2026, 2059, 2068] payload_rows_per_dst=[4156, 4080, 4138, 4048, 4040, 4052, 4118, 4136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4156, 4080, 4138, 4048, 4040, 4052, 4118, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28638 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2044, 1954, 2065, 2028, 2006, 2080, 2065, 2142] payload_rows_per_dst=[4088, 3908, 4130, 4056, 4012, 4160, 4130, 4284] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4088, 3908, 4130, 4056, 4012, 4160, 4130, 4284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233570304 dispatch_meta_bytes=114176 combine_payload_bytes=233570304 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233684480 ingress_bytes=2347499520 link_bytes=2347499520 token_bytes=467254784 weight_movement_bytes=2113929216 total_nvlink_bytes=2581184000 weight_to_token_ratio=4.524147 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314501742e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234864640 dispatch_meta_bytes=114680 combine_payload_bytes=234864640 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=234979320 ingress_bytes=4462723072 link_bytes=4462723072 token_bytes=469843960 weight_movement_bytes=4227858432 total_nvlink_bytes=4697702392 weight_to_token_ratio=8.998431 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498938637e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=236011520 dispatch_meta_bytes=115240 combine_payload_bytes=236011520 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=236126760 ingress_bytes=3054583808 link_bytes=3054583808 token_bytes=472138280 weight_movement_bytes=2818572288 total_nvlink_bytes=3290710568 weight_to_token_ratio=5.969803 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710439428e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28624 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2076, 2040, 2039, 2067, 2021, 2072, 2027, 2042] payload_rows_per_dst=[4152, 4080, 4078, 4134, 4042, 4144, 4054, 4084] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4152, 4080, 4078, 4134, 4042, 4144, 4054, 4084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234766224 ingress_bytes=3053223936 link_bytes=3053223936 token_bytes=469417872 weight_movement_bytes=2818572288 total_nvlink_bytes=3287990160 weight_to_token_ratio=6.004399 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709677956e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235126848 ingress_bytes=3053584384 link_bytes=3053584384 token_bytes=470138944 weight_movement_bytes=2818572288 total_nvlink_bytes=3288711232 weight_to_token_ratio=5.995190 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709879792e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114552 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234717048 ingress_bytes=1643888640 link_bytes=1643888640 token_bytes=469319544 weight_movement_bytes=1409286144 total_nvlink_bytes=1878605688 weight_to_token_ratio=3.002829 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205090191e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32712 payload_rows_remote=28536 dup_rows=56 dup_rows_remote=20 dedup_ratio=0.001709 router_tokens_per_expert=[2096, 2067, 2011, 2031, 2076, 2068, 2034, 2001] payload_rows_per_dst=[4124, 4024, 3968, 4088, 4212, 4046, 4074, 4176] dup_rows_per_dst=[0, 0, 0, 6, 8, 6, 0, 36] combine_rows_per_src=[4124, 4024, 3968, 4088, 4212, 4046, 4074, 4176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234487808 dispatch_meta_bytes=114496 combine_payload_bytes=234487808 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234602304 ingress_bytes=1643773952 link_bytes=1643773952 token_bytes=469090112 weight_movement_bytes=1409286144 total_nvlink_bytes=1878376256 weight_to_token_ratio=3.004297 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204447986e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32736 payload_rows_remote=28418 dup_rows=32 dup_rows_remote=18 dedup_ratio=0.000977 router_tokens_per_expert=[2094, 1992, 2079, 2109, 1998, 2055, 2074, 1983] payload_rows_per_dst=[4318, 4058, 4166, 4144, 3996, 4110, 4126, 3818] dup_rows_per_dst=[14, 16, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[4318, 4058, 4166, 4144, 3996, 4110, 4126, 3818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28720 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2083, 2068, 1999, 2067, 2024, 2009, 2072, 2062] payload_rows_per_dst=[4166, 4136, 3998, 4134, 4048, 4018, 4144, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4166, 4136, 3998, 4134, 4048, 4018, 4144, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28624 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1984, 2100, 2062, 1968, 2040, 2124, 2072, 2034] payload_rows_per_dst=[3968, 4200, 4124, 3936, 4080, 4248, 4144, 4068] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3968, 4200, 4124, 3936, 4080, 4248, 4144, 4068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233766912 dispatch_meta_bytes=114224 combine_payload_bytes=233766912 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233881136 ingress_bytes=1643053056 link_bytes=1643053056 token_bytes=467648048 weight_movement_bytes=1409286144 total_nvlink_bytes=1876934192 weight_to_token_ratio=3.013561 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.200411270e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28668 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 2044, 2074, 2050, 2120, 2027, 1984, 2025] payload_rows_per_dst=[4120, 4088, 4148, 4100, 4240, 4054, 3968, 4050] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 4088, 4148, 4100, 4240, 4054, 3968, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2022, 2077, 2047, 2014, 2033, 2069, 2081, 2041] payload_rows_per_dst=[4044, 4154, 4094, 4028, 4066, 4138, 4162, 4082] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4044, 4154, 4094, 4028, 4066, 4138, 4162, 4082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2038, 2104, 2045, 2048, 1995, 2053, 2071] payload_rows_per_dst=[4060, 4076, 4208, 4090, 4096, 3990, 4106, 4142] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4076, 4208, 4090, 4096, 3990, 4106, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235274240 dispatch_meta_bytes=114880 combine_payload_bytes=235274240 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235389120 ingress_bytes=1644560384 link_bytes=1644560384 token_bytes=470663360 weight_movement_bytes=1409286144 total_nvlink_bytes=1879949504 weight_to_token_ratio=2.994255 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208851677e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232800256 dispatch_meta_bytes=113744 combine_payload_bytes=232800256 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232914000 ingress_bytes=3051372544 link_bytes=3051372544 token_bytes=465714256 weight_movement_bytes=2818572288 total_nvlink_bytes=3284286544 weight_to_token_ratio=6.052149 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708641254e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234487808 dispatch_meta_bytes=114496 combine_payload_bytes=234487808 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234602304 ingress_bytes=3053060096 link_bytes=3053060096 token_bytes=469090112 weight_movement_bytes=2818572288 total_nvlink_bytes=3287662400 weight_to_token_ratio=6.008595 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709586213e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28656 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2016, 2081, 2094, 2041, 2027, 2056, 2031, 2038] payload_rows_per_dst=[4032, 4162, 4188, 4082, 4054, 4112, 4062, 4076] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4032, 4162, 4188, 4082, 4054, 4112, 4062, 4076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234848256 dispatch_meta_bytes=114672 combine_payload_bytes=234848256 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234962928 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=469811184 weight_movement_bytes=1409286144 total_nvlink_bytes=1879097328 weight_to_token_ratio=2.999686 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234520344 ingress_bytes=1643692032 link_bytes=1643692032 token_bytes=468926232 weight_movement_bytes=1409286144 total_nvlink_bytes=1878212376 weight_to_token_ratio=3.005347 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203989268e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32632 payload_rows_remote=28478 dup_rows=136 dup_rows_remote=130 dedup_ratio=0.004150 router_tokens_per_expert=[2046, 2018, 2016, 2114, 2009, 2046, 2093, 2042] payload_rows_per_dst=[3732, 3960, 4078, 3744, 4488, 4284, 4192, 4154] dup_rows_per_dst=[0, 0, 8, 0, 70, 52, 0, 6] combine_rows_per_src=[3732, 3960, 4078, 3744, 4488, 4284, 4192, 4154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114240 combine_payload_bytes=233963520 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234077760 ingress_bytes=2347892736 link_bytes=2347892736 token_bytes=468041280 weight_movement_bytes=2113929216 total_nvlink_bytes=2581970496 weight_to_token_ratio=4.516544 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314721926e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234749952 dispatch_meta_bytes=114624 combine_payload_bytes=234749952 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234864576 ingress_bytes=2348679168 link_bytes=2348679168 token_bytes=469614528 weight_movement_bytes=2113929216 total_nvlink_bytes=2583543744 weight_to_token_ratio=4.501414 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315162295e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2069, 2058, 2006, 2071, 2002, 2018, 2072, 2088] payload_rows_per_dst=[4138, 4116, 4012, 4142, 4004, 4036, 4144, 4176] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4138, 4116, 4012, 4142, 4004, 4036, 4144, 4176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=233291776 dispatch_meta_bytes=114432 combine_payload_bytes=233291776 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233406208 ingress_bytes=3051864064 link_bytes=3051864064 token_bytes=466697984 weight_movement_bytes=2818572288 total_nvlink_bytes=3285270272 weight_to_token_ratio=6.039392 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708916485e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32654 payload_rows_remote=28500 dup_rows=114 dup_rows_remote=100 dedup_ratio=0.003479 router_tokens_per_expert=[2041, 2034, 2061, 2018, 2087, 2027, 2001, 2115] payload_rows_per_dst=[4154, 4324, 4168, 3948, 4198, 3952, 3952, 3958] dup_rows_per_dst=[14, 24, 12, 26, 8, 26, 4, 0] combine_rows_per_src=[4154, 4324, 4168, 3948, 4198, 3952, 3952, 3958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28750 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2005, 2103, 2024, 2148, 2032, 1999, 2009, 2064] payload_rows_per_dst=[4010, 4206, 4048, 4296, 4064, 3998, 4018, 4128] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4010, 4206, 4048, 4296, 4064, 3998, 4018, 4128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2075, 2048, 2003, 2049, 2074, 2054, 2057] payload_rows_per_dst=[4048, 4150, 4096, 4006, 4098, 4148, 4108, 4114] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4150, 4096, 4006, 4098, 4148, 4108, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235634688 dispatch_meta_bytes=115056 combine_payload_bytes=235634688 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235749744 ingress_bytes=2349563904 link_bytes=2349563904 token_bytes=471384432 weight_movement_bytes=2113929216 total_nvlink_bytes=2585313648 weight_to_token_ratio=4.484512 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315657711e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28566 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1987, 2101, 1999, 2012, 2078, 2121, 2101, 1985] payload_rows_per_dst=[3974, 4202, 3998, 4024, 4156, 4242, 4202, 3970] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3974, 4202, 3998, 4024, 4156, 4242, 4202, 3970] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28594 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2083, 2026, 2087, 2113, 1949, 2010, 2026, 2090] payload_rows_per_dst=[4166, 4052, 4174, 4226, 3898, 4020, 4052, 4180] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4166, 4052, 4174, 4226, 3898, 4020, 4052, 4180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28666 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2063, 2031, 2108, 1977, 2067, 2051, 2040, 2047] payload_rows_per_dst=[4126, 4062, 4216, 3954, 4134, 4102, 4080, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4126, 4062, 4216, 3954, 4134, 4102, 4080, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233472000 dispatch_meta_bytes=114400 combine_payload_bytes=233472000 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233586400 ingress_bytes=1642758144 link_bytes=1642758144 token_bytes=467058400 weight_movement_bytes=1409286144 total_nvlink_bytes=1876344544 weight_to_token_ratio=3.017366 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.198759885e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235520000 dispatch_meta_bytes=115000 combine_payload_bytes=235520000 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235635000 ingress_bytes=1644806144 link_bytes=1644806144 token_bytes=471155000 weight_movement_bytes=1409286144 total_nvlink_bytes=1880441144 weight_to_token_ratio=2.991131 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210227831e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235618304 dispatch_meta_bytes=115048 combine_payload_bytes=235618304 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=235733352 ingress_bytes=5168119808 link_bytes=5168119808 token_bytes=471351656 weight_movement_bytes=4932501504 total_nvlink_bytes=5403853160 weight_to_token_ratio=10.464589 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893931365e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=28614 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.001831 router_tokens_per_expert=[1978, 2041, 2122, 2047, 2079, 2052, 2022, 2043] payload_rows_per_dst=[3956, 3814, 4236, 4094, 4118, 4134, 4262, 4094] dup_rows_per_dst=[0, 0, 0, 0, 26, 0, 34, 0] combine_rows_per_src=[3956, 3814, 4236, 4094, 4118, 4134, 4262, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234012672 dispatch_meta_bytes=114264 combine_payload_bytes=234012672 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234126936 ingress_bytes=3757228032 link_bytes=3757228032 token_bytes=468139608 weight_movement_bytes=3523215360 total_nvlink_bytes=3991354968 weight_to_token_ratio=7.525993 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103890864e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234242048 dispatch_meta_bytes=114376 combine_payload_bytes=234242048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234356424 ingress_bytes=1643528192 link_bytes=1643528192 token_bytes=468598472 weight_movement_bytes=1409286144 total_nvlink_bytes=1877884616 weight_to_token_ratio=3.007449 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203071833e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234831872 dispatch_meta_bytes=114664 combine_payload_bytes=234831872 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234946536 ingress_bytes=3758047232 link_bytes=3758047232 token_bytes=469778408 weight_movement_bytes=3523215360 total_nvlink_bytes=3992993768 weight_to_token_ratio=7.499739 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104349581e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114696 combine_payload_bytes=234405888 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=234520584 ingress_bytes=5871550464 link_bytes=5871550464 token_bytes=468926472 weight_movement_bytes=5637144576 total_nvlink_bytes=6106071048 weight_to_token_ratio=12.021383 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.287823169e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2075, 2071, 1976, 2027, 2104, 2048, 2028] payload_rows_per_dst=[4110, 4150, 4142, 3952, 4054, 4208, 4096, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4150, 4142, 3952, 4054, 4208, 4096, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32692 payload_rows_remote=28546 dup_rows=76 dup_rows_remote=70 dedup_ratio=0.002319 router_tokens_per_expert=[2035, 1972, 2080, 2085, 2095, 2027, 2054, 2036] payload_rows_per_dst=[4146, 3966, 4310, 4222, 4186, 3836, 4020, 4006] dup_rows_per_dst=[6, 2, 48, 20, 0, 0, 0, 0] combine_rows_per_src=[4146, 3966, 4310, 4222, 4186, 3836, 4020, 4006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1988, 2092, 2034, 2075, 2032, 2089, 2060, 2014] payload_rows_per_dst=[3976, 4184, 4068, 4150, 4064, 4178, 4120, 4028] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3976, 4184, 4068, 4150, 4064, 4178, 4120, 4028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28644 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2109, 2018, 2011, 2062, 2034, 2056, 2031, 2063] payload_rows_per_dst=[4218, 4036, 4022, 4124, 4068, 4112, 4062, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4218, 4036, 4022, 4124, 4068, 4112, 4062, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235225088 dispatch_meta_bytes=114856 combine_payload_bytes=235225088 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235339944 ingress_bytes=3053797376 link_bytes=3053797376 token_bytes=470565032 weight_movement_bytes=2818572288 total_nvlink_bytes=3289137320 weight_to_token_ratio=5.989761 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709999059e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2081, 2055, 2061, 2041, 2032, 1997, 2037, 2080] payload_rows_per_dst=[4162, 4110, 4122, 4082, 4064, 3994, 4074, 4160] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4162, 4110, 4122, 4082, 4064, 3994, 4074, 4160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28710 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2051, 2029, 2030, 2026, 2071, 2009, 2077] payload_rows_per_dst=[4182, 4102, 4058, 4060, 4052, 4142, 4018, 4154] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4102, 4058, 4060, 4052, 4142, 4018, 4154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234799008 ingress_bytes=2348613632 link_bytes=2348613632 token_bytes=469483424 weight_movement_bytes=2113929216 total_nvlink_bytes=2583412640 weight_to_token_ratio=4.502671 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315125598e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233848832 dispatch_meta_bytes=114464 combine_payload_bytes=233848832 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233963296 ingress_bytes=3052421120 link_bytes=3052421120 token_bytes=467812128 weight_movement_bytes=2818572288 total_nvlink_bytes=3286384416 weight_to_token_ratio=6.025009 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709228413e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234651648 dispatch_meta_bytes=114576 combine_payload_bytes=234651648 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234766224 ingress_bytes=2348580864 link_bytes=2348580864 token_bytes=469417872 weight_movement_bytes=2113929216 total_nvlink_bytes=2583347088 weight_to_token_ratio=4.503299 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315107249e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2027, 2037, 1973, 2100, 2046, 2032, 2090, 2079] payload_rows_per_dst=[4054, 4074, 3946, 4200, 4092, 4064, 4180, 4158] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4054, 4074, 3946, 4200, 4092, 4064, 4180, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32680 payload_rows_remote=28392 dup_rows=88 dup_rows_remote=82 dedup_ratio=0.002686 router_tokens_per_expert=[2044, 2049, 2036, 2089, 2091, 2011, 1983, 2081] payload_rows_per_dst=[3720, 4328, 3952, 4218, 4120, 4006, 4048, 4288] dup_rows_per_dst=[0, 46, 0, 6, 0, 20, 10, 6] combine_rows_per_src=[3720, 4328, 3952, 4218, 4120, 4006, 4048, 4288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234880968 ingress_bytes=2348695552 link_bytes=2348695552 token_bytes=469647304 weight_movement_bytes=2113929216 total_nvlink_bytes=2583576520 weight_to_token_ratio=4.501099 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315171470e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235192320 dispatch_meta_bytes=114840 combine_payload_bytes=235192320 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235307160 ingress_bytes=1644478464 link_bytes=1644478464 token_bytes=470499480 weight_movement_bytes=1409286144 total_nvlink_bytes=1879785624 weight_to_token_ratio=2.995298 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208392959e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235257984 ingress_bytes=1644429312 link_bytes=1644429312 token_bytes=470401152 weight_movement_bytes=1409286144 total_nvlink_bytes=1879687296 weight_to_token_ratio=2.995924 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208117729e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28472 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1988, 2003, 2021, 2107, 2148, 2082, 1965, 2070] payload_rows_per_dst=[3976, 4006, 4042, 4214, 4296, 4164, 3930, 4140] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3976, 4006, 4042, 4214, 4296, 4164, 3930, 4140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=232587264 dispatch_meta_bytes=113896 combine_payload_bytes=232587264 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232701160 ingress_bytes=2346516480 link_bytes=2346516480 token_bytes=465288424 weight_movement_bytes=2113929216 total_nvlink_bytes=2579217640 weight_to_token_ratio=4.543266 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313951280e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28592 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2078, 2052, 2044, 2011, 2035, 2045, 2088, 2031] payload_rows_per_dst=[4156, 4104, 4088, 4022, 4070, 4090, 4176, 4062] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4156, 4104, 4088, 4022, 4070, 4090, 4176, 4062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28558 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2100, 2005, 1966, 2105, 2031, 2076, 2030, 2071] payload_rows_per_dst=[4200, 4010, 3932, 4210, 4062, 4152, 4060, 4142] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4200, 4010, 3932, 4210, 4062, 4152, 4060, 4142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32714 payload_rows_remote=28552 dup_rows=54 dup_rows_remote=38 dedup_ratio=0.001648 router_tokens_per_expert=[2015, 2033, 2073, 2045, 2046, 2062, 2079, 2031] payload_rows_per_dst=[4162, 4072, 4252, 4028, 3920, 4238, 4146, 3896] dup_rows_per_dst=[16, 0, 14, 6, 0, 18, 0, 0] combine_rows_per_src=[4162, 4072, 4252, 4028, 3920, 4238, 4146, 3896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7519502696 prefetch_bytes=20434649088 reduce_bytes=16911433728 weight_movement_bytes=37346082816 egress_bytes=3760669032 ingress_bytes=41104916480 link_bytes=41104916480 total_nvlink_bytes=44865585512 weight_to_token_ratio=4.966563 prefetch_share_of_ingress=0.497134 dedup_ratio=0.000000 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=24 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.438564839e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233242624 dispatch_meta_bytes=113888 combine_payload_bytes=233242624 prefetch_bytes=1409286144 reduce_bytes=2818572288 egress_bytes=233356512 ingress_bytes=4461101056 link_bytes=4461101056 token_bytes=466599136 weight_movement_bytes=4227858432 total_nvlink_bytes=4694457568 weight_to_token_ratio=9.061008 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498030376e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28762 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2003, 2002, 2071, 2088, 2038, 2055, 2078] payload_rows_per_dst=[4098, 4006, 4004, 4142, 4176, 4076, 4110, 4156] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4006, 4004, 4142, 4176, 4076, 4110, 4156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28684 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2062, 2042, 2001, 2050, 2027, 2093, 2043] payload_rows_per_dst=[4132, 4124, 4084, 4002, 4100, 4054, 4186, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4124, 4084, 4002, 4100, 4054, 4186, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234225664 dispatch_meta_bytes=114368 combine_payload_bytes=234225664 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234340032 ingress_bytes=2348154880 link_bytes=2348154880 token_bytes=468565696 weight_movement_bytes=2113929216 total_nvlink_bytes=2582494912 weight_to_token_ratio=4.511489 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314868716e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233947136 dispatch_meta_bytes=114232 combine_payload_bytes=233947136 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234061368 ingress_bytes=2347876352 link_bytes=2347876352 token_bytes=468008504 weight_movement_bytes=2113929216 total_nvlink_bytes=2581937720 weight_to_token_ratio=4.516861 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314712752e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233897984 dispatch_meta_bytes=114360 combine_payload_bytes=233897984 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234012344 ingress_bytes=2347827200 link_bytes=2347827200 token_bytes=467910328 weight_movement_bytes=2113929216 total_nvlink_bytes=2581839544 weight_to_token_ratio=4.517808 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314685229e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32690 payload_rows_remote=28540 dup_rows=78 dup_rows_remote=74 dedup_ratio=0.002380 router_tokens_per_expert=[2003, 2042, 2034, 2118, 2045, 2046, 2033, 2063] payload_rows_per_dst=[3940, 4138, 4068, 3810, 4074, 4156, 4354, 4150] dup_rows_per_dst=[0, 12, 0, 0, 2, 12, 48, 4] combine_rows_per_src=[3940, 4138, 4068, 3810, 4074, 4156, 4354, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7522124776 prefetch_bytes=19730006016 reduce_bytes=19730006016 weight_movement_bytes=39460012032 egress_bytes=3761980392 ingress_bytes=43220156416 link_bytes=43220156416 total_nvlink_bytes=46982136808 weight_to_token_ratio=5.245860 prefetch_share_of_ingress=0.456500 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=28 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.512592718e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235618304 dispatch_meta_bytes=115048 combine_payload_bytes=235618304 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235733352 ingress_bytes=3054190592 link_bytes=3054190592 token_bytes=471351656 weight_movement_bytes=2818572288 total_nvlink_bytes=3289923944 weight_to_token_ratio=5.979765 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710219243e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234979328 dispatch_meta_bytes=114736 combine_payload_bytes=234979328 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235094064 ingress_bytes=2348908544 link_bytes=2348908544 token_bytes=470073392 weight_movement_bytes=2113929216 total_nvlink_bytes=2584002608 weight_to_token_ratio=4.497020 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315290736e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7471456752 prefetch_bytes=20434649088 reduce_bytes=19025362944 weight_movement_bytes=39460012032 egress_bytes=3736642032 ingress_bytes=43194826752 link_bytes=43194826752 total_nvlink_bytes=46931468784 weight_to_token_ratio=5.281435 prefetch_share_of_ingress=0.473081 dedup_ratio=0.002239 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=27 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.511706246e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233799680 dispatch_meta_bytes=114456 combine_payload_bytes=233799680 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233914136 ingress_bytes=3052371968 link_bytes=3052371968 token_bytes=467713816 weight_movement_bytes=2818572288 total_nvlink_bytes=3286286104 weight_to_token_ratio=6.026275 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709200890e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2058, 2007, 2042, 2070, 2026, 2036, 2086] payload_rows_per_dst=[4118, 4116, 4014, 4084, 4140, 4052, 4072, 4172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4116, 4014, 4084, 4140, 4052, 4072, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28774 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2072, 2036, 2113, 1997, 2094, 2043, 1977, 2052] payload_rows_per_dst=[4144, 4072, 4226, 3994, 4188, 4086, 3954, 4104] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4144, 4072, 4226, 3994, 4188, 4086, 3954, 4104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7459068144 prefetch_bytes=19025362944 reduce_bytes=22548578304 weight_movement_bytes=41573941248 egress_bytes=3730446576 ingress_bytes=45302562816 link_bytes=45302562816 total_nvlink_bytes=49033009392 weight_to_token_ratio=5.573611 prefetch_share_of_ingress=0.419962 dedup_ratio=0.002796 prefetch_slots=54 unused_prefetch_slots=10 slot_utilization=0.843750 reduce_slots=32 remote_demand=27 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017519 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.585471510e-04 prefetch_bytes_per_gemm_flop=6.658380682e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28792 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2086, 1988, 1998, 2081, 2021, 2150, 2046, 2014] payload_rows_per_dst=[4172, 3976, 3996, 4162, 4042, 4300, 4092, 4028] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4172, 3976, 3996, 4162, 4042, 4300, 4092, 4028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235192416 ingress_bytes=2349006848 link_bytes=2349006848 token_bytes=470270048 weight_movement_bytes=2113929216 total_nvlink_bytes=2584199264 weight_to_token_ratio=4.495139 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315345783e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235716608 dispatch_meta_bytes=115096 combine_payload_bytes=235716608 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=235831704 ingress_bytes=4463575040 link_bytes=4463575040 token_bytes=471548312 weight_movement_bytes=4227858432 total_nvlink_bytes=4699406744 weight_to_token_ratio=8.965907 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499415704e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28560 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 2033, 2104, 2120, 2074, 1974, 1996, 2028] payload_rows_per_dst=[4110, 4066, 4208, 4240, 4148, 3948, 3992, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 4066, 4208, 4240, 4148, 3948, 3992, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235864064 dispatch_meta_bytes=115168 combine_payload_bytes=235864064 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235979232 ingress_bytes=2349793280 link_bytes=2349793280 token_bytes=471843296 weight_movement_bytes=2113929216 total_nvlink_bytes=2585772512 weight_to_token_ratio=4.480151 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315786152e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7522878624 prefetch_bytes=17616076800 reduce_bytes=18320719872 weight_movement_bytes=35936796672 egress_bytes=3762357408 ingress_bytes=39697317888 link_bytes=39697317888 total_nvlink_bytes=43459675296 weight_to_token_ratio=4.777001 prefetch_share_of_ingress=0.443760 dedup_ratio=0.000000 prefetch_slots=50 unused_prefetch_slots=14 slot_utilization=0.781250 reduce_slots=26 remote_demand=25 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.389302561e-04 prefetch_bytes_per_gemm_flop=6.165167298e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7526910072 prefetch_bytes=20434649088 reduce_bytes=20434649088 weight_movement_bytes=40869298176 egress_bytes=3764373624 ingress_bytes=44631834624 link_bytes=44631834624 total_nvlink_bytes=48396208248 weight_to_token_ratio=5.429758 prefetch_share_of_ingress=0.457849 dedup_ratio=0.000000 prefetch_slots=58 unused_prefetch_slots=6 slot_utilization=0.906250 reduce_slots=29 remote_demand=29 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016572 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.561997773e-04 prefetch_bytes_per_gemm_flop=7.151594066e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233963520 dispatch_meta_bytes=114240 combine_payload_bytes=233963520 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234077760 ingress_bytes=1643249664 link_bytes=1643249664 token_bytes=468041280 weight_movement_bytes=1409286144 total_nvlink_bytes=1877327424 weight_to_token_ratio=3.011030 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201512192e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7516126768 prefetch_bytes=18320719872 reduce_bytes=16911433728 weight_movement_bytes=35232153600 egress_bytes=3758980656 ingress_bytes=38989299712 link_bytes=38989299712 total_nvlink_bytes=42748280368 weight_to_token_ratio=4.687541 prefetch_share_of_ingress=0.469891 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=24 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.364523772e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28556 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2124, 2106, 1999, 2096, 1954, 2015, 2046, 2044] payload_rows_per_dst=[4248, 4212, 3998, 4192, 3908, 4030, 4092, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4248, 4212, 3998, 4192, 3908, 4030, 4092, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114224 combine_payload_bytes=233930752 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234044976 ingress_bytes=1643216896 link_bytes=1643216896 token_bytes=467975728 weight_movement_bytes=1409286144 total_nvlink_bytes=1877261872 weight_to_token_ratio=3.011451 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201328705e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7513766896 prefetch_bytes=18320719872 reduce_bytes=20434649088 weight_movement_bytes=38755368960 egress_bytes=3757800432 ingress_bytes=42511335424 link_bytes=42511335424 total_nvlink_bytes=46269135856 weight_to_token_ratio=5.157915 prefetch_share_of_ingress=0.430961 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=29 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.487785833e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none - [2026-09-16 21:35:56] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 575.4 | throughput per GPU (TFLOP/s/GPU): 579.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.505587E+00 | loss scale: 1.0 | grad norm: 3.072 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2061, 1982, 2026, 2116, 2026, 2157, 2030, 1986] payload_rows_per_dst=[4122, 3964, 4052, 4232, 4052, 4314, 4060, 3972] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4122, 3964, 4052, 4232, 4052, 4314, 4060, 3972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32722 payload_rows_remote=28468 dup_rows=46 dup_rows_remote=32 dedup_ratio=0.001404 router_tokens_per_expert=[2086, 2046, 2082, 2025, 2041, 2022, 2062, 2020] payload_rows_per_dst=[4254, 4094, 4176, 4032, 4224, 3778, 4124, 4040] dup_rows_per_dst=[14, 0, 6, 0, 26, 0, 0, 0] combine_rows_per_src=[4254, 4094, 4176, 4032, 4224, 3778, 4124, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28782 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2066, 2076, 2009, 1980, 1993, 2061, 2127, 2072] payload_rows_per_dst=[4132, 4152, 4018, 3960, 3986, 4122, 4254, 4144] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4132, 4152, 4018, 3960, 3986, 4122, 4254, 4144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28630 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2067, 1967, 2059, 2068, 2077, 2038, 2069, 2039] payload_rows_per_dst=[4134, 3934, 4118, 4136, 4154, 4076, 4138, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4134, 3934, 4118, 4136, 4154, 4076, 4138, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235356336 ingress_bytes=2349170688 link_bytes=2349170688 token_bytes=470597808 weight_movement_bytes=2113929216 total_nvlink_bytes=2584527024 weight_to_token_ratio=4.492008 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315437526e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=233209856 dispatch_meta_bytes=114000 combine_payload_bytes=233209856 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233323856 ingress_bytes=2347139072 link_bytes=2347139072 token_bytes=466533712 weight_movement_bytes=2113929216 total_nvlink_bytes=2580462928 weight_to_token_ratio=4.531139 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314299906e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28622 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2073, 2087, 2014, 2054, 1995, 2073, 2116, 1972] payload_rows_per_dst=[4146, 4174, 4028, 4108, 3990, 4146, 4232, 3944] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 4174, 4028, 4108, 3990, 4146, 4232, 3944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=28274 dup_rows=92 dup_rows_remote=38 dedup_ratio=0.002808 router_tokens_per_expert=[1988, 2061, 2099, 2020, 2015, 2133, 2034, 2034] payload_rows_per_dst=[3756, 4300, 4144, 4040, 4008, 4088, 3938, 4402] dup_rows_per_dst=[0, 32, 0, 6, 0, 0, 0, 54] combine_rows_per_src=[3756, 4300, 4144, 4040, 4008, 4088, 3938, 4402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235782144 dispatch_meta_bytes=115128 combine_payload_bytes=235782144 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235897272 ingress_bytes=3054354432 link_bytes=3054354432 token_bytes=471679416 weight_movement_bytes=2818572288 total_nvlink_bytes=3290251704 weight_to_token_ratio=5.975610 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710310987e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1992, 2023, 2090, 2054, 2058, 2072, 2047, 2048] payload_rows_per_dst=[3984, 4046, 4180, 4108, 4116, 4144, 4094, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3984, 4046, 4180, 4108, 4116, 4144, 4094, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28696 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2113, 2036, 2023, 2073, 2106, 2050, 1964, 2019] payload_rows_per_dst=[4226, 4072, 4046, 4146, 4212, 4100, 3928, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4226, 4072, 4046, 4146, 4212, 4100, 3928, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114520 combine_payload_bytes=234536960 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234651480 ingress_bytes=2348466176 link_bytes=2348466176 token_bytes=469188440 weight_movement_bytes=2113929216 total_nvlink_bytes=2583117656 weight_to_token_ratio=4.505501 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315043029e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28702 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2158, 2014, 2033, 2040, 1990, 2035, 2047, 2067] payload_rows_per_dst=[4316, 4028, 4066, 4080, 3980, 4070, 4094, 4134] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4316, 4028, 4066, 4080, 3980, 4070, 4094, 4134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234471424 dispatch_meta_bytes=114488 combine_payload_bytes=234471424 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234585912 ingress_bytes=4462329856 link_bytes=4462329856 token_bytes=469057336 weight_movement_bytes=4227858432 total_nvlink_bytes=4696915768 weight_to_token_ratio=9.013522 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498718453e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32700 payload_rows_remote=28618 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.002075 router_tokens_per_expert=[2041, 2050, 2027, 2110, 2039, 2017, 2003, 2097] payload_rows_per_dst=[4082, 4186, 3980, 4340, 4208, 4034, 3972, 3898] dup_rows_per_dst=[0, 22, 0, 14, 32, 0, 0, 0] combine_rows_per_src=[4082, 4186, 3980, 4340, 4208, 4034, 3972, 3898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=231620608 dispatch_meta_bytes=113248 combine_payload_bytes=231620608 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=231733856 ingress_bytes=1640906752 link_bytes=1640906752 token_bytes=463354464 weight_movement_bytes=1409286144 total_nvlink_bytes=1872640608 weight_to_token_ratio=3.041486 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.188392863e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114640 combine_payload_bytes=234782720 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234897360 ingress_bytes=3053355008 link_bytes=3053355008 token_bytes=469680080 weight_movement_bytes=2818572288 total_nvlink_bytes=3288252368 weight_to_token_ratio=6.001047 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709751351e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235126784 dispatch_meta_bytes=114808 combine_payload_bytes=235126784 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235241592 ingress_bytes=2349056000 link_bytes=2349056000 token_bytes=470368376 weight_movement_bytes=2113929216 total_nvlink_bytes=2584297592 weight_to_token_ratio=4.494199 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315373306e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=235077632 dispatch_meta_bytes=114784 combine_payload_bytes=235077632 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235192416 ingress_bytes=2349006848 link_bytes=2349006848 token_bytes=470270048 weight_movement_bytes=2113929216 total_nvlink_bytes=2584199264 weight_to_token_ratio=4.495139 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315345783e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2095, 1959, 2040, 2040, 2101, 2081, 2017] payload_rows_per_dst=[4102, 4190, 3918, 4080, 4080, 4202, 4162, 4034] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4190, 3918, 4080, 4080, 4202, 4162, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234438656 dispatch_meta_bytes=114744 combine_payload_bytes=234438656 prefetch_bytes=0 reduce_bytes=4932501504 egress_bytes=234553400 ingress_bytes=5166940160 link_bytes=5166940160 token_bytes=468992056 weight_movement_bytes=4932501504 total_nvlink_bytes=5401493560 weight_to_token_ratio=10.517239 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893270811e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28710 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2116, 2022, 2109, 2044, 2025, 2010, 2029, 2029] payload_rows_per_dst=[4232, 4044, 4218, 4088, 4050, 4020, 4058, 4058] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4232, 4044, 4218, 4088, 4050, 4020, 4058, 4058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235012096 dispatch_meta_bytes=114752 combine_payload_bytes=235012096 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235126848 ingress_bytes=1644298240 link_bytes=1644298240 token_bytes=470138944 weight_movement_bytes=1409286144 total_nvlink_bytes=1879425088 weight_to_token_ratio=2.997595 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207383780e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 1980, 1977, 2091, 2087, 2084, 2099, 2019] payload_rows_per_dst=[4094, 3960, 3954, 4182, 4174, 4168, 4198, 4038] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 3960, 3954, 4182, 4174, 4168, 4198, 4038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28774 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2029, 2104, 2028, 2012, 2098, 1997, 2027, 2089] payload_rows_per_dst=[4058, 4208, 4056, 4024, 4196, 3994, 4054, 4178] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4058, 4208, 4056, 4024, 4196, 3994, 4054, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235192320 dispatch_meta_bytes=114840 combine_payload_bytes=235192320 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235307160 ingress_bytes=2349121536 link_bytes=2349121536 token_bytes=470499480 weight_movement_bytes=2113929216 total_nvlink_bytes=2584428696 weight_to_token_ratio=4.492947 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315410003e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32638 payload_rows_remote=28436 dup_rows=130 dup_rows_remote=116 dedup_ratio=0.003967 router_tokens_per_expert=[2120, 2028, 2026, 2044, 1989, 2058, 2052, 2067] payload_rows_per_dst=[3750, 3752, 4320, 4030, 4124, 4282, 4178, 4202] dup_rows_per_dst=[0, 0, 40, 0, 20, 36, 20, 14] combine_rows_per_src=[3750, 3752, 4320, 4030, 4124, 4282, 4178, 4202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28588 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2063, 2090, 2074, 2052, 1995, 1989, 2061, 2060] payload_rows_per_dst=[4126, 4180, 4148, 4104, 3990, 3978, 4122, 4120] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4126, 4180, 4148, 4104, 3990, 3978, 4122, 4120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28582 dup_rows=84 dup_rows_remote=80 dedup_ratio=0.002563 router_tokens_per_expert=[2048, 2034, 2072, 2052, 2046, 2051, 2045, 2036] payload_rows_per_dst=[4102, 4068, 4320, 4358, 4082, 3818, 3926, 4010] dup_rows_per_dst=[4, 0, 24, 38, 0, 18, 0, 0] combine_rows_per_src=[4102, 4068, 4320, 4358, 4082, 3818, 3926, 4010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28760 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2051, 2079, 2011, 2004, 2044, 2076, 2054, 2065] payload_rows_per_dst=[4102, 4158, 4022, 4008, 4088, 4152, 4108, 4130] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4102, 4158, 4022, 4008, 4088, 4152, 4108, 4130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=236044288 dispatch_meta_bytes=115256 combine_payload_bytes=236044288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236159544 ingress_bytes=1645330432 link_bytes=1645330432 token_bytes=472203832 weight_movement_bytes=1409286144 total_nvlink_bytes=1881489976 weight_to_token_ratio=2.984487 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213163624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235716608 dispatch_meta_bytes=115096 combine_payload_bytes=235716608 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235831704 ingress_bytes=2349645824 link_bytes=2349645824 token_bytes=471548312 weight_movement_bytes=2113929216 total_nvlink_bytes=2585477528 weight_to_token_ratio=4.482954 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315703582e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=232947712 dispatch_meta_bytes=114208 combine_payload_bytes=232947712 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233061920 ingress_bytes=3051520000 link_bytes=3051520000 token_bytes=466009632 weight_movement_bytes=2818572288 total_nvlink_bytes=3284581920 weight_to_token_ratio=6.048313 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708723823e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28676 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2095, 2006, 2081, 1969, 2046, 2034, 2034, 2119] payload_rows_per_dst=[4190, 4012, 4162, 3938, 4092, 4068, 4068, 4238] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4190, 4012, 4162, 3938, 4092, 4068, 4068, 4238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=234192896 dispatch_meta_bytes=114352 combine_payload_bytes=234192896 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234307248 ingress_bytes=2348122112 link_bytes=2348122112 token_bytes=468500144 weight_movement_bytes=2113929216 total_nvlink_bytes=2582429360 weight_to_token_ratio=4.512121 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314850367e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234143744 dispatch_meta_bytes=114648 combine_payload_bytes=234143744 prefetch_bytes=704643072 reduce_bytes=4932501504 egress_bytes=234258392 ingress_bytes=5871288320 link_bytes=5871288320 token_bytes=468402136 weight_movement_bytes=5637144576 total_nvlink_bytes=6105546712 weight_to_token_ratio=12.034840 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=3.287676380e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=235601920 dispatch_meta_bytes=115040 combine_payload_bytes=235601920 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235716960 ingress_bytes=2349531136 link_bytes=2349531136 token_bytes=471318880 weight_movement_bytes=2113929216 total_nvlink_bytes=2585248096 weight_to_token_ratio=4.485136 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315639362e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2089, 2098, 2060, 1961, 2007, 2073, 2055, 2041] payload_rows_per_dst=[4178, 4196, 4120, 3922, 4014, 4146, 4110, 4082] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4178, 4196, 4120, 3922, 4014, 4146, 4110, 4082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234913792 dispatch_meta_bytes=114704 combine_payload_bytes=234913792 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235028496 ingress_bytes=2348843008 link_bytes=2348843008 token_bytes=469942288 weight_movement_bytes=2113929216 total_nvlink_bytes=2583871504 weight_to_token_ratio=4.498274 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315254039e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28772 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2052, 2058, 1998, 2123, 2067, 2051, 1978, 2057] payload_rows_per_dst=[4104, 4116, 3996, 4246, 4134, 4102, 3956, 4114] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4104, 4116, 3996, 4246, 4134, 4102, 3956, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234880968 ingress_bytes=3053338624 link_bytes=3053338624 token_bytes=469647304 weight_movement_bytes=2818572288 total_nvlink_bytes=3288219592 weight_to_token_ratio=6.001466 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709742177e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32682 payload_rows_remote=28502 dup_rows=86 dup_rows_remote=58 dedup_ratio=0.002625 router_tokens_per_expert=[2095, 2013, 1994, 2071, 2072, 2038, 2110, 1991] payload_rows_per_dst=[3772, 4148, 4156, 4142, 4086, 4128, 4070, 4180] dup_rows_per_dst=[0, 20, 30, 0, 0, 8, 0, 28] combine_rows_per_src=[3772, 4148, 4156, 4142, 4086, 4128, 4070, 4180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28542 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2073, 2041, 1948, 2084, 2011, 2113, 2104, 2010] payload_rows_per_dst=[4146, 4082, 3896, 4168, 4022, 4226, 4208, 4020] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4146, 4082, 3896, 4168, 4022, 4226, 4208, 4020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235700224 dispatch_meta_bytes=115088 combine_payload_bytes=235700224 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235815312 ingress_bytes=1644986368 link_bytes=1644986368 token_bytes=471515536 weight_movement_bytes=1409286144 total_nvlink_bytes=1880801680 weight_to_token_ratio=2.988843 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211237010e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32730 payload_rows_remote=28606 dup_rows=38 dup_rows_remote=34 dedup_ratio=0.001160 router_tokens_per_expert=[2044, 2103, 2059, 2043, 2019, 2064, 2053, 1999] payload_rows_per_dst=[4124, 4410, 4180, 4082, 4018, 4128, 4064, 3724] dup_rows_per_dst=[4, 20, 10, 0, 0, 0, 4, 0] combine_rows_per_src=[4124, 4410, 4180, 4082, 4018, 4128, 4064, 3724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28672 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2048, 2021, 2048, 2084, 2072, 2034, 2023] payload_rows_per_dst=[4108, 4096, 4042, 4096, 4168, 4144, 4068, 4046] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4096, 4042, 4096, 4168, 4144, 4068, 4046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28844 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2087, 2093, 2032, 1962, 2065, 2022, 2060, 2063] payload_rows_per_dst=[4174, 4186, 4064, 3924, 4130, 4044, 4120, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4174, 4186, 4064, 3924, 4130, 4044, 4120, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28786 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2076, 2012, 2054, 2075, 1991, 2003, 2104, 2069] payload_rows_per_dst=[4152, 4024, 4108, 4150, 3982, 4006, 4208, 4138] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4152, 4024, 4108, 4150, 3982, 4006, 4208, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233488384 dispatch_meta_bytes=114240 combine_payload_bytes=233488384 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233602624 ingress_bytes=2347417600 link_bytes=2347417600 token_bytes=467091008 weight_movement_bytes=2113929216 total_nvlink_bytes=2581020224 weight_to_token_ratio=4.525733 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314455870e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=233816064 dispatch_meta_bytes=114168 combine_payload_bytes=233816064 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233930232 ingress_bytes=3052388352 link_bytes=3052388352 token_bytes=467746296 weight_movement_bytes=2818572288 total_nvlink_bytes=3286318584 weight_to_token_ratio=6.025857 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709210064e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28636 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 1988, 2076, 2060, 2004, 2074, 2066, 2062] payload_rows_per_dst=[4108, 3976, 4152, 4120, 4008, 4148, 4132, 4124] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 3976, 4152, 4120, 4008, 4148, 4132, 4124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114560 combine_payload_bytes=234340352 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234454912 ingress_bytes=3052912640 link_bytes=3052912640 token_bytes=468795264 weight_movement_bytes=2818572288 total_nvlink_bytes=3287367552 weight_to_token_ratio=6.012374 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709503644e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=234881024 dispatch_meta_bytes=114688 combine_payload_bytes=234881024 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234995712 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=469876736 weight_movement_bytes=1409286144 total_nvlink_bytes=1879162880 weight_to_token_ratio=2.999268 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=236290048 dispatch_meta_bytes=115376 combine_payload_bytes=236290048 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236405424 ingress_bytes=1645576192 link_bytes=1645576192 token_bytes=472695472 weight_movement_bytes=1409286144 total_nvlink_bytes=1881981616 weight_to_token_ratio=2.981383 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.214539778e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235814912 dispatch_meta_bytes=115144 combine_payload_bytes=235814912 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235930056 ingress_bytes=3054387200 link_bytes=3054387200 token_bytes=471744968 weight_movement_bytes=2818572288 total_nvlink_bytes=3290317256 weight_to_token_ratio=5.974780 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710329336e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28718 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2021, 2099, 2025, 1964, 2109, 2028, 2084, 2054] payload_rows_per_dst=[4042, 4198, 4050, 3928, 4218, 4056, 4168, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4042, 4198, 4050, 3928, 4218, 4056, 4168, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234586112 dispatch_meta_bytes=114544 combine_payload_bytes=234586112 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234700656 ingress_bytes=2348515328 link_bytes=2348515328 token_bytes=469286768 weight_movement_bytes=2113929216 total_nvlink_bytes=2583215984 weight_to_token_ratio=4.504557 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315070552e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235257856 dispatch_meta_bytes=114872 combine_payload_bytes=235257856 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235372728 ingress_bytes=3053830144 link_bytes=3053830144 token_bytes=470630584 weight_movement_bytes=2818572288 total_nvlink_bytes=3289202872 weight_to_token_ratio=5.988927 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710017407e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32688 payload_rows_remote=28356 dup_rows=80 dup_rows_remote=66 dedup_ratio=0.002441 router_tokens_per_expert=[2069, 1982, 2003, 2047, 2108, 2071, 1997, 2107] payload_rows_per_dst=[4002, 3964, 4006, 3788, 4190, 4316, 4090, 4332] dup_rows_per_dst=[0, 0, 0, 2, 20, 28, 16, 14] combine_rows_per_src=[4002, 3964, 4006, 3788, 4190, 4316, 4090, 4332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28814 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2050, 2059, 2096, 2060, 2050, 1977, 2068, 2024] payload_rows_per_dst=[4100, 4118, 4192, 4120, 4100, 3954, 4136, 4048] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4100, 4118, 4192, 4120, 4100, 3954, 4136, 4048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32698 payload_rows_remote=28506 dup_rows=70 dup_rows_remote=54 dedup_ratio=0.002136 router_tokens_per_expert=[2012, 2039, 2089, 2057, 2102, 1981, 2059, 2045] payload_rows_per_dst=[4192, 4144, 4218, 4196, 3976, 4054, 3868, 4050] dup_rows_per_dst=[16, 8, 22, 10, 0, 14, 0, 0] combine_rows_per_src=[4192, 4144, 4218, 4196, 3976, 4054, 3868, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28806 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2067, 1981, 2005, 2063, 2066, 2093, 2033, 2076] payload_rows_per_dst=[4134, 3962, 4010, 4126, 4132, 4186, 4066, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4134, 3962, 4010, 4126, 4132, 4186, 4066, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28776 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2089, 2028, 2005, 1996, 2066, 2054, 2101] payload_rows_per_dst=[4090, 4178, 4056, 4010, 3992, 4132, 4108, 4202] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4178, 4056, 4010, 3992, 4132, 4108, 4202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28634 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2020, 2089, 2057, 2067, 2092, 1950, 2048, 2061] payload_rows_per_dst=[4040, 4178, 4114, 4134, 4184, 3900, 4096, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4040, 4178, 4114, 4134, 4184, 3900, 4096, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2036, 2007, 2096, 2114, 2035, 2002, 2035] payload_rows_per_dst=[4118, 4072, 4014, 4192, 4228, 4070, 4004, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4072, 4014, 4192, 4228, 4070, 4004, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=232292352 dispatch_meta_bytes=113688 combine_payload_bytes=232292352 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=232406040 ingress_bytes=3050864640 link_bytes=3050864640 token_bytes=464698392 weight_movement_bytes=2818572288 total_nvlink_bytes=3283270680 weight_to_token_ratio=6.065380 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708356849e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=236044288 dispatch_meta_bytes=115256 combine_payload_bytes=236044288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236159544 ingress_bytes=1645330432 link_bytes=1645330432 token_bytes=472203832 weight_movement_bytes=1409286144 total_nvlink_bytes=1881489976 weight_to_token_ratio=2.984487 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.213163624e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233521152 dispatch_meta_bytes=114240 combine_payload_bytes=233521152 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=233635392 ingress_bytes=3052093440 link_bytes=3052093440 token_bytes=467156544 weight_movement_bytes=2818572288 total_nvlink_bytes=3285728832 weight_to_token_ratio=6.033464 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709044926e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=235978752 dispatch_meta_bytes=115224 combine_payload_bytes=235978752 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236093976 ingress_bytes=2349907968 link_bytes=2349907968 token_bytes=472072728 weight_movement_bytes=2113929216 total_nvlink_bytes=2586001944 weight_to_token_ratio=4.477974 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315850372e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235732992 dispatch_meta_bytes=115104 combine_payload_bytes=235732992 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235848096 ingress_bytes=3758948352 link_bytes=3758948352 token_bytes=471581088 weight_movement_bytes=3523215360 total_nvlink_bytes=3994796448 weight_to_token_ratio=7.471070 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104854171e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=234569728 dispatch_meta_bytes=114536 combine_payload_bytes=234569728 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=234684264 ingress_bytes=4462428160 link_bytes=4462428160 token_bytes=469253992 weight_movement_bytes=4227858432 total_nvlink_bytes=4697112424 weight_to_token_ratio=9.009744 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.498773499e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28562 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2047, 2099, 2103, 2005, 2037, 1994, 2054, 2045] payload_rows_per_dst=[4094, 4198, 4206, 4010, 4074, 3988, 4108, 4090] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4094, 4198, 4206, 4010, 4074, 3988, 4108, 4090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=235634688 dispatch_meta_bytes=115056 combine_payload_bytes=235634688 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235749744 ingress_bytes=3054206976 link_bytes=3054206976 token_bytes=471384432 weight_movement_bytes=2818572288 total_nvlink_bytes=3289956720 weight_to_token_ratio=5.979350 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710228418e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233979904 dispatch_meta_bytes=114248 combine_payload_bytes=233979904 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234094152 ingress_bytes=3052552192 link_bytes=3052552192 token_bytes=468074056 weight_movement_bytes=2818572288 total_nvlink_bytes=3286646344 weight_to_token_ratio=6.021637 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709301808e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32712 payload_rows_remote=28502 dup_rows=56 dup_rows_remote=40 dedup_ratio=0.001709 router_tokens_per_expert=[2027, 2023, 2004, 2099, 2054, 2059, 2050, 2068] payload_rows_per_dst=[4014, 3810, 3918, 4198, 4130, 4132, 4300, 4210] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 36, 16] combine_rows_per_src=[4014, 3810, 3918, 4198, 4130, 4132, 4300, 4210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28564 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2040, 2083, 2102, 2013, 2008, 2102, 2037, 1999] payload_rows_per_dst=[4080, 4166, 4204, 4026, 4016, 4204, 4074, 3998] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4080, 4166, 4204, 4026, 4016, 4204, 4074, 3998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32708 payload_rows_remote=28524 dup_rows=60 dup_rows_remote=50 dedup_ratio=0.001831 router_tokens_per_expert=[2069, 1952, 2054, 2132, 2017, 2105, 2011, 2044] payload_rows_per_dst=[4184, 4112, 4114, 4120, 3930, 4188, 4122, 3938] dup_rows_per_dst=[10, 40, 2, 0, 0, 0, 8, 0] combine_rows_per_src=[4184, 4112, 4114, 4120, 3930, 4188, 4122, 3938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28674 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 2047, 2035, 2045, 2039, 2052, 2029, 2077] payload_rows_per_dst=[4120, 4094, 4070, 4090, 4078, 4104, 4058, 4154] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 4094, 4070, 4090, 4078, 4104, 4058, 4154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2048, 2049, 2108, 2059, 2063, 2001, 2076, 1980] payload_rows_per_dst=[4096, 4098, 4216, 4118, 4126, 4002, 4152, 3960] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 4098, 4216, 4118, 4126, 4002, 4152, 3960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28590 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2020, 2044, 2058, 2113, 2089, 2032, 1996, 2032] payload_rows_per_dst=[4040, 4088, 4116, 4226, 4178, 4064, 3992, 4064] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4040, 4088, 4116, 4226, 4178, 4064, 3992, 4064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28778 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2055, 1953, 2116, 2082, 2086, 2069, 1995, 2028] payload_rows_per_dst=[4110, 3906, 4232, 4164, 4172, 4138, 3990, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4110, 3906, 4232, 4164, 4172, 4138, 3990, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233488384 dispatch_meta_bytes=114168 combine_payload_bytes=233488384 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233602552 ingress_bytes=2347417600 link_bytes=2347417600 token_bytes=467090936 weight_movement_bytes=2113929216 total_nvlink_bytes=2581020152 weight_to_token_ratio=4.525734 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314455870e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=233996288 dispatch_meta_bytes=114256 combine_payload_bytes=233996288 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234110544 ingress_bytes=1643282432 link_bytes=1643282432 token_bytes=468106832 weight_movement_bytes=1409286144 total_nvlink_bytes=1877392976 weight_to_token_ratio=3.010608 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201695679e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=233668608 dispatch_meta_bytes=114296 combine_payload_bytes=233668608 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233782904 ingress_bytes=3756883968 link_bytes=3756883968 token_bytes=467451512 weight_movement_bytes=3523215360 total_nvlink_bytes=3990666872 weight_to_token_ratio=7.537071 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103698202e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28730 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2077, 2061, 2019, 1985, 2037, 2067, 2094, 2044] payload_rows_per_dst=[4154, 4122, 4038, 3970, 4074, 4134, 4188, 4088] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4154, 4122, 4038, 3970, 4074, 4134, 4188, 4088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234209280 dispatch_meta_bytes=114360 combine_payload_bytes=234209280 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234323640 ingress_bytes=2348138496 link_bytes=2348138496 token_bytes=468532920 weight_movement_bytes=2113929216 total_nvlink_bytes=2582462136 weight_to_token_ratio=4.511805 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314859542e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234897408 dispatch_meta_bytes=114696 combine_payload_bytes=234897408 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235012104 ingress_bytes=2348826624 link_bytes=2348826624 token_bytes=469909512 weight_movement_bytes=2113929216 total_nvlink_bytes=2583838728 weight_to_token_ratio=4.498588 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315244865e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=234700800 dispatch_meta_bytes=114600 combine_payload_bytes=234700800 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234815400 ingress_bytes=2348630016 link_bytes=2348630016 token_bytes=469516200 weight_movement_bytes=2113929216 total_nvlink_bytes=2583445416 weight_to_token_ratio=4.502356 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315134772e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235749376 dispatch_meta_bytes=115112 combine_payload_bytes=235749376 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235864488 ingress_bytes=1645035520 link_bytes=1645035520 token_bytes=471613864 weight_movement_bytes=1409286144 total_nvlink_bytes=1880900008 weight_to_token_ratio=2.988220 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.211512240e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235356160 dispatch_meta_bytes=114920 combine_payload_bytes=235356160 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235471080 ingress_bytes=1644642304 link_bytes=1644642304 token_bytes=470827240 weight_movement_bytes=1409286144 total_nvlink_bytes=1880113384 weight_to_token_ratio=2.993213 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209310395e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32734 payload_rows_remote=28638 dup_rows=34 dup_rows_remote=32 dedup_ratio=0.001038 router_tokens_per_expert=[2039, 2066, 2070, 2066, 2025, 2046, 2032, 2040] payload_rows_per_dst=[4034, 4010, 4052, 4152, 4110, 4104, 4176, 4096] dup_rows_per_dst=[0, 0, 0, 4, 14, 4, 10, 2] combine_rows_per_src=[4034, 4010, 4052, 4152, 4110, 4104, 4176, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28590 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2083, 1980, 1989, 2089, 2089, 2064, 2056] payload_rows_per_dst=[4068, 4166, 3960, 3978, 4178, 4178, 4128, 4112] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4166, 3960, 3978, 4178, 4178, 4128, 4112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32602 payload_rows_remote=28268 dup_rows=166 dup_rows_remote=116 dedup_ratio=0.005066 router_tokens_per_expert=[2069, 2006, 2009, 2048, 1997, 2029, 2079, 2147] payload_rows_per_dst=[4334, 3894, 4254, 4314, 3974, 3736, 4502, 3594] dup_rows_per_dst=[50, 0, 30, 28, 0, 0, 58, 0] combine_rows_per_src=[4334, 3894, 4254, 4314, 3974, 3736, 4502, 3594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28680 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2006, 2029, 2010, 2055, 2044, 2096, 2003, 2141] payload_rows_per_dst=[4012, 4058, 4020, 4110, 4088, 4192, 4006, 4282] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4012, 4058, 4020, 4110, 4088, 4192, 4006, 4282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28612 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2058, 2078, 2041, 2057, 1994, 2020, 2068, 2068] payload_rows_per_dst=[4116, 4156, 4082, 4114, 3988, 4040, 4136, 4136] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4116, 4156, 4082, 4114, 3988, 4040, 4136, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2030, 2104, 2045, 2049, 2046, 2019, 2018, 2073] payload_rows_per_dst=[4060, 4208, 4090, 4098, 4092, 4038, 4036, 4146] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4060, 4208, 4090, 4098, 4092, 4038, 4036, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28768 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2056, 2046, 2002, 2000, 2062, 2032, 2049, 2137] payload_rows_per_dst=[4112, 4092, 4004, 4000, 4124, 4064, 4098, 4274] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4112, 4092, 4004, 4000, 4124, 4064, 4098, 4274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234602496 dispatch_meta_bytes=114680 combine_payload_bytes=234602496 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234717176 ingress_bytes=3757817856 link_bytes=3757817856 token_bytes=469319672 weight_movement_bytes=3523215360 total_nvlink_bytes=3992535032 weight_to_token_ratio=7.507069 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104221140e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28712 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2064, 2106, 2028, 2034, 2036, 2054, 1988, 2074] payload_rows_per_dst=[4128, 4212, 4056, 4068, 4072, 4108, 3976, 4148] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4128, 4212, 4056, 4068, 4072, 4108, 3976, 4148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234209280 dispatch_meta_bytes=114360 combine_payload_bytes=234209280 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234323640 ingress_bytes=2348138496 link_bytes=2348138496 token_bytes=468532920 weight_movement_bytes=2113929216 total_nvlink_bytes=2582462136 weight_to_token_ratio=4.511805 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314859542e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=231571456 dispatch_meta_bytes=113536 combine_payload_bytes=231571456 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=231684992 ingress_bytes=2345500672 link_bytes=2345500672 token_bytes=463256448 weight_movement_bytes=2113929216 total_nvlink_bytes=2577185664 weight_to_token_ratio=4.563194 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313382470e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234946560 dispatch_meta_bytes=114720 combine_payload_bytes=234946560 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235061280 ingress_bytes=3053518848 link_bytes=3053518848 token_bytes=470007840 weight_movement_bytes=2818572288 total_nvlink_bytes=3288580128 weight_to_token_ratio=5.996862 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709843095e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=234389504 dispatch_meta_bytes=114448 combine_payload_bytes=234389504 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234503952 ingress_bytes=2348318720 link_bytes=2348318720 token_bytes=468893456 weight_movement_bytes=2113929216 total_nvlink_bytes=2582822672 weight_to_token_ratio=4.508336 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314960460e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235487472 ingress_bytes=2349301760 link_bytes=2349301760 token_bytes=470860016 weight_movement_bytes=2113929216 total_nvlink_bytes=2584789232 weight_to_token_ratio=4.489507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315510921e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=235667456 dispatch_meta_bytes=115072 combine_payload_bytes=235667456 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235782528 ingress_bytes=2349596672 link_bytes=2349596672 token_bytes=471449984 weight_movement_bytes=2113929216 total_nvlink_bytes=2585379200 weight_to_token_ratio=4.483889 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315676059e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235208704 dispatch_meta_bytes=114848 combine_payload_bytes=235208704 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235323552 ingress_bytes=2349137920 link_bytes=2349137920 token_bytes=470532256 weight_movement_bytes=2113929216 total_nvlink_bytes=2584461472 weight_to_token_ratio=4.492634 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315419177e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32698 payload_rows_remote=28380 dup_rows=70 dup_rows_remote=54 dedup_ratio=0.002136 router_tokens_per_expert=[2012, 2017, 2005, 2066, 2075, 2023, 2071, 2115] payload_rows_per_dst=[3848, 3984, 4164, 3932, 4262, 4150, 4040, 4318] dup_rows_per_dst=[0, 0, 22, 0, 16, 16, 0, 16] combine_rows_per_src=[3848, 3984, 4164, 3932, 4262, 4150, 4040, 4318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28592 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2078, 2089, 1979, 2100, 2063, 2088, 1926, 2061] payload_rows_per_dst=[4156, 4178, 3958, 4200, 4126, 4176, 3852, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4156, 4178, 3958, 4200, 4126, 4176, 3852, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32720 payload_rows_remote=28546 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.001465 router_tokens_per_expert=[2082, 2056, 2072, 2038, 2057, 2018, 2072, 1989] payload_rows_per_dst=[4174, 4120, 4144, 4238, 4104, 4036, 4216, 3688] dup_rows_per_dst=[0, 2, 0, 38, 0, 0, 8, 0] combine_rows_per_src=[4174, 4120, 4144, 4238, 4104, 4036, 4216, 3688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28748 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2093, 2103, 2057, 2002, 2010, 2010, 2018, 2091] payload_rows_per_dst=[4186, 4206, 4114, 4004, 4020, 4020, 4036, 4182] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4186, 4206, 4114, 4004, 4020, 4020, 4036, 4182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2032, 2094, 2016, 2038, 2064, 2049, 2018, 2073] payload_rows_per_dst=[4064, 4188, 4032, 4076, 4128, 4098, 4036, 4146] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4064, 4188, 4032, 4076, 4128, 4098, 4036, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28706 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2054, 2001, 2053, 2001, 2057, 2132, 2031, 2055] payload_rows_per_dst=[4108, 4002, 4106, 4002, 4114, 4264, 4062, 4110] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4108, 4002, 4106, 4002, 4114, 4264, 4062, 4110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2045, 2068, 2080, 2020, 2004, 2114, 2025, 2028] payload_rows_per_dst=[4090, 4136, 4160, 4040, 4008, 4228, 4050, 4056] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4090, 4136, 4160, 4040, 4008, 4228, 4050, 4056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=232488960 dispatch_meta_bytes=113736 combine_payload_bytes=232488960 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=232602696 ingress_bytes=3755704320 link_bytes=3755704320 token_bytes=465091656 weight_movement_bytes=3523215360 total_nvlink_bytes=3988307016 weight_to_token_ratio=7.575314 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103037648e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28514 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2083, 2013, 2127, 2083, 2005, 1986, 2006, 2081] payload_rows_per_dst=[4166, 4026, 4254, 4166, 4010, 3972, 4012, 4162] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4166, 4026, 4254, 4166, 4010, 3972, 4012, 4162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234225664 dispatch_meta_bytes=114368 combine_payload_bytes=234225664 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234340032 ingress_bytes=3052797952 link_bytes=3052797952 token_bytes=468565696 weight_movement_bytes=2818572288 total_nvlink_bytes=3287137984 weight_to_token_ratio=6.015319 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709439423e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233848832 dispatch_meta_bytes=114376 combine_payload_bytes=233848832 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233963208 ingress_bytes=3052421120 link_bytes=3052421120 token_bytes=467812040 weight_movement_bytes=2818572288 total_nvlink_bytes=3286384328 weight_to_token_ratio=6.025010 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709228413e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235503616 dispatch_meta_bytes=114992 combine_payload_bytes=235503616 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235618608 ingress_bytes=3054075904 link_bytes=3054075904 token_bytes=471122224 weight_movement_bytes=2818572288 total_nvlink_bytes=3289694512 weight_to_token_ratio=5.982677 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710155023e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=234127360 dispatch_meta_bytes=114320 combine_payload_bytes=234127360 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234241680 ingress_bytes=3052699648 link_bytes=3052699648 token_bytes=468369040 weight_movement_bytes=2818572288 total_nvlink_bytes=3286941328 weight_to_token_ratio=6.017845 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709384377e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235159552 dispatch_meta_bytes=114824 combine_payload_bytes=235159552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235274376 ingress_bytes=1644445696 link_bytes=1644445696 token_bytes=470433928 weight_movement_bytes=1409286144 total_nvlink_bytes=1879720072 weight_to_token_ratio=2.995715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208209472e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235454688 ingress_bytes=2349268992 link_bytes=2349268992 token_bytes=470794464 weight_movement_bytes=2113929216 total_nvlink_bytes=2584723680 weight_to_token_ratio=4.490132 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315492572e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233586688 dispatch_meta_bytes=114056 combine_payload_bytes=233586688 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233700744 ingress_bytes=3052158976 link_bytes=3052158976 token_bytes=467287432 weight_movement_bytes=2818572288 total_nvlink_bytes=3285859720 weight_to_token_ratio=6.031774 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709081623e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32672 payload_rows_remote=28558 dup_rows=96 dup_rows_remote=88 dedup_ratio=0.002930 router_tokens_per_expert=[2089, 2123, 1993, 2008, 2049, 2002, 2076, 2044] payload_rows_per_dst=[4040, 4000, 3874, 4206, 4106, 4138, 4194, 4114] dup_rows_per_dst=[0, 0, 6, 30, 6, 30, 16, 8] combine_rows_per_src=[4040, 4000, 3874, 4206, 4106, 4138, 4194, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28652 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2077, 2038, 1994, 2075, 2058, 2064, 2054] payload_rows_per_dst=[4048, 4154, 4076, 3988, 4150, 4116, 4128, 4108] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4154, 4076, 3988, 4150, 4116, 4128, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28660 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1909, 2061, 2085, 2141, 2054, 2074, 1999, 2061] payload_rows_per_dst=[3818, 4122, 4170, 4282, 4108, 4148, 3998, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3818, 4122, 4170, 4282, 4108, 4148, 3998, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32682 payload_rows_remote=28362 dup_rows=86 dup_rows_remote=66 dedup_ratio=0.002625 router_tokens_per_expert=[2088, 1973, 2010, 2124, 2039, 2065, 2019, 2066] payload_rows_per_dst=[4320, 3956, 4062, 4158, 4128, 4130, 4038, 3890] dup_rows_per_dst=[20, 4, 22, 0, 40, 0, 0, 0] combine_rows_per_src=[4320, 3956, 4062, 4158, 4128, 4130, 4038, 3890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28724 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2022, 2060, 2066, 2079, 1987, 2056, 2076] payload_rows_per_dst=[4076, 4044, 4120, 4132, 4158, 3974, 4112, 4152] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4044, 4120, 4132, 4158, 3974, 4112, 4152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28910 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2095, 2030, 2081, 2037, 2020, 2058, 1929, 2134] payload_rows_per_dst=[4190, 4060, 4162, 4074, 4040, 4116, 3858, 4268] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4190, 4060, 4162, 4074, 4040, 4116, 3858, 4268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=233947136 dispatch_meta_bytes=114584 combine_payload_bytes=233947136 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234061720 ingress_bytes=3757162496 link_bytes=3757162496 token_bytes=468008856 weight_movement_bytes=3523215360 total_nvlink_bytes=3991224216 weight_to_token_ratio=7.528095 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103854166e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28706 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2004, 2014, 2136, 2031, 2019, 2041, 2010, 2129] payload_rows_per_dst=[4008, 4028, 4272, 4062, 4038, 4082, 4020, 4258] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4008, 4028, 4272, 4062, 4038, 4082, 4020, 4258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=234717184 dispatch_meta_bytes=114608 combine_payload_bytes=234717184 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=234831792 ingress_bytes=1644003328 link_bytes=1644003328 token_bytes=469548976 weight_movement_bytes=1409286144 total_nvlink_bytes=1878835120 weight_to_token_ratio=3.001361 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205732396e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28742 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2111, 2030, 2013, 2078, 2094, 2027, 2092, 1939] payload_rows_per_dst=[4222, 4060, 4026, 4156, 4188, 4054, 4184, 3878] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4222, 4060, 4026, 4156, 4188, 4054, 4184, 3878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=114640 combine_payload_bytes=234782720 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234897360 ingress_bytes=3053355008 link_bytes=3053355008 token_bytes=469680080 weight_movement_bytes=2818572288 total_nvlink_bytes=3288252368 weight_to_token_ratio=6.001047 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709751351e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=232341504 dispatch_meta_bytes=113712 combine_payload_bytes=232341504 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232455216 ingress_bytes=1641627648 link_bytes=1641627648 token_bytes=464796720 weight_movement_bytes=1409286144 total_nvlink_bytes=1874082864 weight_to_token_ratio=3.032048 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.192429580e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=236830720 dispatch_meta_bytes=115640 combine_payload_bytes=236830720 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236946360 ingress_bytes=1646116864 link_bytes=1646116864 token_bytes=473777080 weight_movement_bytes=1409286144 total_nvlink_bytes=1883063224 weight_to_token_ratio=2.974576 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.217567315e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235307008 dispatch_meta_bytes=114896 combine_payload_bytes=235307008 prefetch_bytes=704643072 reduce_bytes=3523215360 egress_bytes=235421904 ingress_bytes=4463165440 link_bytes=4463165440 token_bytes=470728912 weight_movement_bytes=4227858432 total_nvlink_bytes=4698587344 weight_to_token_ratio=8.981514 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.499186345e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=235159552 dispatch_meta_bytes=114824 combine_payload_bytes=235159552 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235274376 ingress_bytes=1644445696 link_bytes=1644445696 token_bytes=470433928 weight_movement_bytes=1409286144 total_nvlink_bytes=1879720072 weight_to_token_ratio=2.995715 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208209472e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235454464 dispatch_meta_bytes=114968 combine_payload_bytes=235454464 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=235569432 ingress_bytes=3054026752 link_bytes=3054026752 token_bytes=471023896 weight_movement_bytes=2818572288 total_nvlink_bytes=3289596184 weight_to_token_ratio=5.983926 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710127500e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32654 payload_rows_remote=28402 dup_rows=114 dup_rows_remote=84 dedup_ratio=0.003479 router_tokens_per_expert=[2022, 2130, 2029, 1986, 2044, 2029, 2095, 2049] payload_rows_per_dst=[4044, 4180, 3720, 3816, 4014, 4038, 4590, 4252] dup_rows_per_dst=[0, 0, 0, 0, 0, 6, 78, 30] combine_rows_per_src=[4044, 4180, 3720, 3816, 4014, 4038, 4590, 4252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2067, 2042, 2126, 2021, 2027, 2024, 2094, 1983] payload_rows_per_dst=[4134, 4084, 4252, 4042, 4054, 4048, 4188, 3966] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4134, 4084, 4252, 4042, 4054, 4048, 4188, 3966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28732 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2039, 2092, 2080, 2045, 2018, 2038, 2007] payload_rows_per_dst=[4130, 4078, 4184, 4160, 4090, 4036, 4076, 4014] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4078, 4184, 4160, 4090, 4036, 4076, 4014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=232669184 dispatch_meta_bytes=113944 combine_payload_bytes=232669184 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232783128 ingress_bytes=2346598400 link_bytes=2346598400 token_bytes=465452312 weight_movement_bytes=2113929216 total_nvlink_bytes=2579381528 weight_to_token_ratio=4.541667 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313997152e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28700 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2095, 2033, 2047, 1990, 2103, 2075, 2034, 2007] payload_rows_per_dst=[4190, 4066, 4094, 3980, 4206, 4150, 4068, 4014] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4190, 4066, 4094, 3980, 4206, 4150, 4068, 4014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32706 payload_rows_remote=28446 dup_rows=62 dup_rows_remote=38 dedup_ratio=0.001892 router_tokens_per_expert=[2066, 2027, 2108, 2028, 2057, 2016, 2076, 2006] payload_rows_per_dst=[4260, 4062, 4082, 4254, 4154, 4032, 3850, 4012] dup_rows_per_dst=[24, 0, 6, 24, 8, 0, 0, 0] combine_rows_per_src=[4260, 4062, 4082, 4254, 4154, 4032, 3850, 4012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28590 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2033, 2089, 2048, 1950, 2114, 2024, 2073, 2053] payload_rows_per_dst=[4066, 4178, 4096, 3900, 4228, 4048, 4146, 4106] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4066, 4178, 4096, 3900, 4228, 4048, 4146, 4106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28556 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2007, 2082, 2049, 2106, 2055, 2137, 1986, 1962] payload_rows_per_dst=[4014, 4164, 4098, 4212, 4110, 4274, 3972, 3924] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4014, 4164, 4098, 4212, 4110, 4274, 3972, 3924] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235225088 dispatch_meta_bytes=114856 combine_payload_bytes=235225088 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235339944 ingress_bytes=2349154304 link_bytes=2349154304 token_bytes=470565032 weight_movement_bytes=2113929216 total_nvlink_bytes=2584494248 weight_to_token_ratio=4.492321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315428352e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28648 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2100, 2060, 2018, 1976, 2080, 2024, 2084] payload_rows_per_dst=[4084, 4200, 4120, 4036, 3952, 4160, 4048, 4168] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4200, 4120, 4036, 3952, 4160, 4048, 4168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=235372544 dispatch_meta_bytes=114928 combine_payload_bytes=235372544 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235487472 ingress_bytes=2349301760 link_bytes=2349301760 token_bytes=470860016 weight_movement_bytes=2113929216 total_nvlink_bytes=2584789232 weight_to_token_ratio=4.489507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315510921e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=233029632 dispatch_meta_bytes=113936 combine_payload_bytes=233029632 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233143568 ingress_bytes=3756244992 link_bytes=3756244992 token_bytes=466173200 weight_movement_bytes=3523215360 total_nvlink_bytes=3989388560 weight_to_token_ratio=7.557739 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103340402e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235110400 dispatch_meta_bytes=114800 combine_payload_bytes=235110400 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=235225200 ingress_bytes=5167611904 link_bytes=5167611904 token_bytes=470335600 weight_movement_bytes=4932501504 total_nvlink_bytes=5402837104 weight_to_token_ratio=10.487196 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893646960e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=234209280 dispatch_meta_bytes=114360 combine_payload_bytes=234209280 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234323640 ingress_bytes=2348138496 link_bytes=2348138496 token_bytes=468532920 weight_movement_bytes=2113929216 total_nvlink_bytes=2582462136 weight_to_token_ratio=4.511805 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314859542e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=233930752 dispatch_meta_bytes=114224 combine_payload_bytes=233930752 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=234044976 ingress_bytes=3757146112 link_bytes=3757146112 token_bytes=467975728 weight_movement_bytes=3523215360 total_nvlink_bytes=3991191088 weight_to_token_ratio=7.528628 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103844992e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32584 payload_rows_remote=28028 dup_rows=184 dup_rows_remote=94 dedup_ratio=0.005615 router_tokens_per_expert=[2064, 2060, 2066, 2071, 2007, 2029, 2097, 1990] payload_rows_per_dst=[3436, 4204, 4076, 4142, 3842, 4380, 3948, 4556] dup_rows_per_dst=[0, 14, 0, 0, 6, 70, 4, 90] combine_rows_per_src=[3436, 4204, 4076, 4142, 3842, 4380, 3948, 4556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234684416 dispatch_meta_bytes=114592 combine_payload_bytes=234684416 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234799008 ingress_bytes=2348613632 link_bytes=2348613632 token_bytes=469483424 weight_movement_bytes=2113929216 total_nvlink_bytes=2583412640 weight_to_token_ratio=4.502671 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315125598e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=229605376 dispatch_meta_bytes=112488 combine_payload_bytes=229605376 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=229717864 ingress_bytes=1638891520 link_bytes=1638891520 token_bytes=459323240 weight_movement_bytes=1409286144 total_nvlink_bytes=1868609384 weight_to_token_ratio=3.068179 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.177108405e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28876 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2060, 2107, 2038, 2022, 2058, 1946, 2133, 2020] payload_rows_per_dst=[4120, 4214, 4076, 4044, 4116, 3892, 4266, 4040] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4120, 4214, 4076, 4044, 4116, 3892, 4266, 4040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32680 payload_rows_remote=28462 dup_rows=88 dup_rows_remote=74 dedup_ratio=0.002686 router_tokens_per_expert=[2044, 2027, 2080, 2127, 2116, 1922, 2052, 2016] payload_rows_per_dst=[4218, 4240, 4006, 4168, 4350, 3778, 4128, 3792] dup_rows_per_dst=[14, 28, 0, 0, 14, 0, 32, 0] combine_rows_per_src=[4218, 4240, 4006, 4168, 4350, 3778, 4128, 3792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28580 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2066, 2035, 1992, 2094, 2075, 2051, 2037] payload_rows_per_dst=[4068, 4132, 4070, 3984, 4188, 4150, 4102, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4132, 4070, 3984, 4188, 4150, 4102, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28722 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2117, 2023, 2044, 2055, 2041, 2011, 2050, 2043] payload_rows_per_dst=[4234, 4046, 4088, 4110, 4082, 4022, 4100, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4234, 4046, 4088, 4110, 4082, 4022, 4100, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28742 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2080, 2010, 2053, 2066, 2115, 1999, 2013, 2048] payload_rows_per_dst=[4160, 4020, 4106, 4132, 4230, 3998, 4026, 4096] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4160, 4020, 4106, 4132, 4230, 3998, 4026, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2074, 2009, 2034, 2008, 2056, 2060, 2080, 2063] payload_rows_per_dst=[4148, 4018, 4068, 4016, 4112, 4120, 4160, 4126] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4148, 4018, 4068, 4016, 4112, 4120, 4160, 4126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1988, 2063, 2059, 2110, 2008, 2056, 2065, 2035] payload_rows_per_dst=[3976, 4126, 4118, 4220, 4016, 4112, 4130, 4070] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3976, 4126, 4118, 4220, 4016, 4112, 4130, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=236552192 dispatch_meta_bytes=115504 combine_payload_bytes=236552192 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236667696 ingress_bytes=1645838336 link_bytes=1645838336 token_bytes=473219888 weight_movement_bytes=1409286144 total_nvlink_bytes=1882506032 weight_to_token_ratio=2.978079 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.216007675e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=233160704 dispatch_meta_bytes=114144 combine_payload_bytes=233160704 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233274848 ingress_bytes=2347089920 link_bytes=2347089920 token_bytes=466435552 weight_movement_bytes=2113929216 total_nvlink_bytes=2580364768 weight_to_token_ratio=4.532093 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314272383e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234127360 dispatch_meta_bytes=114320 combine_payload_bytes=234127360 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234241680 ingress_bytes=3052699648 link_bytes=3052699648 token_bytes=468369040 weight_movement_bytes=2818572288 total_nvlink_bytes=3286941328 weight_to_token_ratio=6.017845 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709384377e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235290624 dispatch_meta_bytes=114888 combine_payload_bytes=235290624 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235405512 ingress_bytes=2349219840 link_bytes=2349219840 token_bytes=470696136 weight_movement_bytes=2113929216 total_nvlink_bytes=2584625352 weight_to_token_ratio=4.491070 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315465049e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235454464 dispatch_meta_bytes=114968 combine_payload_bytes=235454464 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235569432 ingress_bytes=3054026752 link_bytes=3054026752 token_bytes=471023896 weight_movement_bytes=2818572288 total_nvlink_bytes=3289596184 weight_to_token_ratio=5.983926 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710127500e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235651392 ingress_bytes=1644822528 link_bytes=1644822528 token_bytes=471187776 weight_movement_bytes=1409286144 total_nvlink_bytes=1880473920 weight_to_token_ratio=2.990923 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210319574e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32670 payload_rows_remote=28660 dup_rows=98 dup_rows_remote=94 dedup_ratio=0.002991 router_tokens_per_expert=[2020, 2014, 2113, 2121, 1976, 2036, 2117, 1987] payload_rows_per_dst=[3880, 3754, 4226, 4242, 3910, 4414, 4234, 4010] dup_rows_per_dst=[0, 14, 0, 0, 0, 80, 0, 4] combine_rows_per_src=[3880, 3754, 4226, 4242, 3910, 4414, 4234, 4010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234700800 dispatch_meta_bytes=114600 combine_payload_bytes=234700800 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234815400 ingress_bytes=3053273088 link_bytes=3053273088 token_bytes=469516200 weight_movement_bytes=2818572288 total_nvlink_bytes=3288088488 weight_to_token_ratio=6.003142 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709705479e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=234782720 dispatch_meta_bytes=115016 combine_payload_bytes=234782720 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=234897736 ingress_bytes=3053355008 link_bytes=3053355008 token_bytes=469680456 weight_movement_bytes=2818572288 total_nvlink_bytes=3288252744 weight_to_token_ratio=6.001042 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709751351e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28764 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2077, 2096, 2035, 1991, 2050, 2002, 2024, 2109] payload_rows_per_dst=[4154, 4192, 4070, 3982, 4100, 4004, 4048, 4218] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4154, 4192, 4070, 3982, 4100, 4004, 4048, 4218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32686 payload_rows_remote=28662 dup_rows=82 dup_rows_remote=82 dedup_ratio=0.002502 router_tokens_per_expert=[2005, 2016, 2059, 2031, 2089, 2012, 2065, 2107] payload_rows_per_dst=[4024, 4402, 4104, 4100, 4178, 4024, 4130, 3724] dup_rows_per_dst=[0, 80, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[4024, 4402, 4104, 4100, 4178, 4024, 4130, 3724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28646 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2128, 1974, 2050, 2055, 2061, 2022, 2027, 2067] payload_rows_per_dst=[4256, 3948, 4100, 4110, 4122, 4044, 4054, 4134] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4256, 3948, 4100, 4110, 4122, 4044, 4054, 4134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28690 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2039, 1995, 2106, 2009, 2020, 2042, 2102] payload_rows_per_dst=[4142, 4078, 3990, 4212, 4018, 4040, 4084, 4204] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4078, 3990, 4212, 4018, 4040, 4084, 4204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2101, 2043, 2041, 2058, 2026, 2023, 2006, 2086] payload_rows_per_dst=[4202, 4086, 4082, 4116, 4052, 4046, 4012, 4172] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4202, 4086, 4082, 4116, 4052, 4046, 4012, 4172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28756 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2055, 2090, 2006, 2003, 2072, 2033, 2101] payload_rows_per_dst=[4048, 4110, 4180, 4012, 4006, 4144, 4066, 4202] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4110, 4180, 4012, 4006, 4144, 4066, 4202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28606 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2065, 2019, 2081, 2033, 2005, 2018, 2074, 2089] payload_rows_per_dst=[4130, 4038, 4162, 4066, 4010, 4036, 4148, 4178] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4130, 4038, 4162, 4066, 4010, 4036, 4148, 4178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235634688 dispatch_meta_bytes=115056 combine_payload_bytes=235634688 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235749744 ingress_bytes=2349563904 link_bytes=2349563904 token_bytes=471384432 weight_movement_bytes=2113929216 total_nvlink_bytes=2585313648 weight_to_token_ratio=4.484512 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315657711e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234668032 dispatch_meta_bytes=114584 combine_payload_bytes=234668032 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234782616 ingress_bytes=2348597248 link_bytes=2348597248 token_bytes=469450648 weight_movement_bytes=2113929216 total_nvlink_bytes=2583379864 weight_to_token_ratio=4.502985 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315116424e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234799104 dispatch_meta_bytes=114976 combine_payload_bytes=234799104 prefetch_bytes=704643072 reduce_bytes=4227858432 egress_bytes=234914080 ingress_bytes=5167300608 link_bytes=5167300608 token_bytes=469713184 weight_movement_bytes=4932501504 total_nvlink_bytes=5402214688 weight_to_token_ratio=10.501092 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893472647e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32660 payload_rows_remote=28442 dup_rows=108 dup_rows_remote=86 dedup_ratio=0.003296 router_tokens_per_expert=[2109, 2006, 2136, 2043, 2010, 2040, 1967, 2073] payload_rows_per_dst=[3988, 4012, 4092, 3914, 4002, 4218, 4216, 4218] dup_rows_per_dst=[0, 0, 0, 14, 10, 20, 42, 22] combine_rows_per_src=[3988, 4012, 4092, 3914, 4002, 4218, 4216, 4218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235684176 ingress_bytes=3054141440 link_bytes=3054141440 token_bytes=471253328 weight_movement_bytes=2818572288 total_nvlink_bytes=3289825616 weight_to_token_ratio=5.981013 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710191720e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=235028480 dispatch_meta_bytes=114760 combine_payload_bytes=235028480 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235143240 ingress_bytes=2348957696 link_bytes=2348957696 token_bytes=470171720 weight_movement_bytes=2113929216 total_nvlink_bytes=2584100936 weight_to_token_ratio=4.496079 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315318259e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235569152 dispatch_meta_bytes=115024 combine_payload_bytes=235569152 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235684176 ingress_bytes=1644855296 link_bytes=1644855296 token_bytes=471253328 weight_movement_bytes=1409286144 total_nvlink_bytes=1880539472 weight_to_token_ratio=2.990507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.210503061e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234340352 dispatch_meta_bytes=114424 combine_payload_bytes=234340352 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234454776 ingress_bytes=1643626496 link_bytes=1643626496 token_bytes=468795128 weight_movement_bytes=1409286144 total_nvlink_bytes=1878081272 weight_to_token_ratio=3.006188 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203622294e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=232996864 dispatch_meta_bytes=114112 combine_payload_bytes=232996864 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=233110976 ingress_bytes=3756212224 link_bytes=3756212224 token_bytes=466107840 weight_movement_bytes=3523215360 total_nvlink_bytes=3989323200 weight_to_token_ratio=7.558799 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103322053e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28716 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2125, 1984, 2035, 2012, 2069, 2026, 2038, 2095] payload_rows_per_dst=[4250, 3968, 4070, 4024, 4138, 4052, 4076, 4190] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4250, 3968, 4070, 4024, 4138, 4052, 4076, 4190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28708 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1979, 2001, 2017, 2113, 2030, 2009, 2121, 2114] payload_rows_per_dst=[3958, 4002, 4034, 4226, 4060, 4018, 4242, 4228] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3958, 4002, 4034, 4226, 4060, 4018, 4242, 4228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32676 payload_rows_remote=28354 dup_rows=92 dup_rows_remote=70 dedup_ratio=0.002808 router_tokens_per_expert=[2055, 2006, 2096, 2023, 2083, 2015, 2057, 2049] payload_rows_per_dst=[4322, 3918, 4214, 4084, 4358, 4144, 3802, 3834] dup_rows_per_dst=[22, 0, 14, 2, 30, 24, 0, 0] combine_rows_per_src=[4322, 3918, 4214, 4084, 4358, 4144, 3802, 3834] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28620 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2042, 2036, 2053, 2074, 2079, 2082, 2035, 1983] payload_rows_per_dst=[4084, 4072, 4106, 4148, 4158, 4164, 4070, 3966] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4084, 4072, 4106, 4148, 4158, 4164, 4070, 3966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28694 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1994, 2037, 2048, 2060, 2046, 2044, 2019, 2136] payload_rows_per_dst=[3988, 4074, 4096, 4120, 4092, 4088, 4038, 4272] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3988, 4074, 4096, 4120, 4092, 4088, 4038, 4272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28774 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1986, 2111, 2060, 2071, 1968, 2099, 1997, 2092] payload_rows_per_dst=[3972, 4222, 4120, 4142, 3936, 4198, 3994, 4184] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3972, 4222, 4120, 4142, 3936, 4198, 3994, 4184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235241472 dispatch_meta_bytes=114864 combine_payload_bytes=235241472 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235356336 ingress_bytes=1644527616 link_bytes=1644527616 token_bytes=470597808 weight_movement_bytes=1409286144 total_nvlink_bytes=1879883952 weight_to_token_ratio=2.994672 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208668190e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235175936 dispatch_meta_bytes=114832 combine_payload_bytes=235175936 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235290768 ingress_bytes=1644462080 link_bytes=1644462080 token_bytes=470466704 weight_movement_bytes=1409286144 total_nvlink_bytes=1879752848 weight_to_token_ratio=2.995507 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.208301216e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2071, 2047, 2032, 1967, 2024, 2038, 2085, 2120] payload_rows_per_dst=[4142, 4094, 4064, 3934, 4048, 4076, 4170, 4240] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4142, 4094, 4064, 3934, 4048, 4076, 4170, 4240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32730 payload_rows_remote=28620 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.001160 router_tokens_per_expert=[2008, 2127, 1996, 2063, 2038, 2072, 2025, 2055] payload_rows_per_dst=[4016, 4196, 3992, 3928, 4238, 4144, 4106, 4110] dup_rows_per_dst=[0, 0, 0, 0, 24, 0, 14, 0] combine_rows_per_src=[4016, 4196, 3992, 3928, 4238, 4144, 4106, 4110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=232275968 dispatch_meta_bytes=113696 combine_payload_bytes=232275968 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=232389664 ingress_bytes=1641562112 link_bytes=1641562112 token_bytes=464665632 weight_movement_bytes=1409286144 total_nvlink_bytes=1873951776 weight_to_token_ratio=3.032904 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.192062605e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=234455040 dispatch_meta_bytes=114480 combine_payload_bytes=234455040 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234569520 ingress_bytes=2348384256 link_bytes=2348384256 token_bytes=469024560 weight_movement_bytes=2113929216 total_nvlink_bytes=2582953776 weight_to_token_ratio=4.507076 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314997157e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=235061248 dispatch_meta_bytes=114776 combine_payload_bytes=235061248 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235176024 ingress_bytes=1644347392 link_bytes=1644347392 token_bytes=470237272 weight_movement_bytes=1409286144 total_nvlink_bytes=1879523416 weight_to_token_ratio=2.996968 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.207659011e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235716608 dispatch_meta_bytes=115096 combine_payload_bytes=235716608 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235831704 ingress_bytes=3054288896 link_bytes=3054288896 token_bytes=471548312 weight_movement_bytes=2818572288 total_nvlink_bytes=3290120600 weight_to_token_ratio=5.977272 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.710274289e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=4 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235257984 ingress_bytes=3053715456 link_bytes=3053715456 token_bytes=470401152 weight_movement_bytes=2818572288 total_nvlink_bytes=3288973440 weight_to_token_ratio=5.991848 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709953187e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234455040 dispatch_meta_bytes=114632 combine_payload_bytes=234455040 prefetch_bytes=0 reduce_bytes=2113929216 egress_bytes=234569672 ingress_bytes=2348384256 link_bytes=2348384256 token_bytes=469024712 weight_movement_bytes=2113929216 total_nvlink_bytes=2582953928 weight_to_token_ratio=4.507074 gemm_rows_defined=16384 gemm_rows_charged=16896 gemm_tail_pad_rows=512 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.030303 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314997157e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28752 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2034, 2070, 2023, 2061, 1993, 2008, 2156, 2039] payload_rows_per_dst=[4068, 4140, 4046, 4122, 3986, 4016, 4312, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4068, 4140, 4046, 4122, 3986, 4016, 4312, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28632 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2014, 2085, 2072, 2072, 2068, 2012, 1986, 2075] payload_rows_per_dst=[4028, 4170, 4144, 4144, 4136, 4024, 3972, 4150] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4028, 4170, 4144, 4144, 4136, 4024, 3972, 4150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28704 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2049, 2071, 2011, 2032, 2049, 2118, 2007, 2047] payload_rows_per_dst=[4098, 4142, 4022, 4064, 4098, 4236, 4014, 4094] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4098, 4142, 4022, 4064, 4098, 4236, 4014, 4094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32648 payload_rows_remote=28326 dup_rows=120 dup_rows_remote=88 dedup_ratio=0.003662 router_tokens_per_expert=[2090, 2013, 2044, 2116, 2033, 2014, 2013, 2061] payload_rows_per_dst=[4322, 4378, 4084, 4102, 4062, 4188, 3846, 3666] dup_rows_per_dst=[32, 50, 18, 0, 0, 20, 0, 0] combine_rows_per_src=[4322, 4378, 4084, 4102, 4062, 4188, 3846, 3666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28658 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2091, 2055, 2082, 2067, 2014, 1976, 2068, 2031] payload_rows_per_dst=[4182, 4110, 4164, 4134, 4028, 3952, 4136, 4062] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4182, 4110, 4164, 4134, 4028, 3952, 4136, 4062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28666 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2059, 2041, 2135, 2009, 2064, 1988, 2051, 2037] payload_rows_per_dst=[4118, 4082, 4270, 4018, 4128, 3976, 4102, 4074] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4118, 4082, 4270, 4018, 4128, 3976, 4102, 4074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235536384 dispatch_meta_bytes=115008 combine_payload_bytes=235536384 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=235651392 ingress_bytes=2349465600 link_bytes=2349465600 token_bytes=471187776 weight_movement_bytes=2113929216 total_nvlink_bytes=2585116992 weight_to_token_ratio=4.486384 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315602664e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234553344 dispatch_meta_bytes=114528 combine_payload_bytes=234553344 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=234667872 ingress_bytes=2348482560 link_bytes=2348482560 token_bytes=469221216 weight_movement_bytes=2113929216 total_nvlink_bytes=2583150432 weight_to_token_ratio=4.505187 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315052203e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28582 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2097, 2014, 2093, 2015, 2058, 2042, 2059, 2006] payload_rows_per_dst=[4194, 4028, 4186, 4030, 4116, 4084, 4118, 4012] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4194, 4028, 4186, 4030, 4116, 4084, 4118, 4012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32716 payload_rows_remote=28498 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.001587 router_tokens_per_expert=[2026, 2008, 2019, 2034, 2032, 2072, 2092, 2101] payload_rows_per_dst=[4014, 3780, 3898, 4068, 4264, 4130, 4344, 4218] dup_rows_per_dst=[0, 0, 0, 0, 34, 0, 18, 0] combine_rows_per_src=[4014, 3780, 3898, 4068, 4264, 4130, 4344, 4218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=235143168 dispatch_meta_bytes=114816 combine_payload_bytes=235143168 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=235257984 ingress_bytes=3053715456 link_bytes=3053715456 token_bytes=470401152 weight_movement_bytes=2818572288 total_nvlink_bytes=3288973440 weight_to_token_ratio=5.991848 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709953187e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=234766336 dispatch_meta_bytes=114632 combine_payload_bytes=234766336 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234880968 ingress_bytes=3053338624 link_bytes=3053338624 token_bytes=469647304 weight_movement_bytes=2818572288 total_nvlink_bytes=3288219592 weight_to_token_ratio=6.001466 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709742177e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=232046592 dispatch_meta_bytes=113656 combine_payload_bytes=232046592 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=232160248 ingress_bytes=2345975808 link_bytes=2345975808 token_bytes=464206840 weight_movement_bytes=2113929216 total_nvlink_bytes=2578136056 weight_to_token_ratio=4.553852 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313648527e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234831872 dispatch_meta_bytes=114664 combine_payload_bytes=234831872 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234946536 ingress_bytes=2348761088 link_bytes=2348761088 token_bytes=469778408 weight_movement_bytes=2113929216 total_nvlink_bytes=2583707624 weight_to_token_ratio=4.499843 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315208167e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234143744 dispatch_meta_bytes=114328 combine_payload_bytes=234143744 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234258072 ingress_bytes=2348072960 link_bytes=2348072960 token_bytes=468401816 weight_movement_bytes=2113929216 total_nvlink_bytes=2582331032 weight_to_token_ratio=4.513068 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314822844e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=233455616 dispatch_meta_bytes=114200 combine_payload_bytes=233455616 prefetch_bytes=704643072 reduce_bytes=2818572288 egress_bytes=233569816 ingress_bytes=3756670976 link_bytes=3756670976 token_bytes=467025432 weight_movement_bytes=3523215360 total_nvlink_bytes=3990240792 weight_to_token_ratio=7.543948 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103578935e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28714 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2032, 2003, 2066, 2067, 2052, 2027, 2030, 2107] payload_rows_per_dst=[4064, 4006, 4132, 4134, 4104, 4054, 4060, 4214] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4064, 4006, 4132, 4134, 4104, 4054, 4060, 4214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28740 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2022, 2020, 2061, 2016, 2014, 2038, 2144, 2069] payload_rows_per_dst=[4044, 4040, 4122, 4032, 4028, 4076, 4288, 4138] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4044, 4040, 4122, 4032, 4028, 4076, 4288, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28614 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1986, 2041, 2055, 2077, 2029, 2123, 2069, 2004] payload_rows_per_dst=[3972, 4082, 4110, 4154, 4058, 4246, 4138, 4008] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3972, 4082, 4110, 4154, 4058, 4246, 4138, 4008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28642 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2038, 2063, 2024, 2107, 1967, 2027, 2059, 2099] payload_rows_per_dst=[4076, 4126, 4048, 4214, 3934, 4054, 4118, 4198] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4076, 4126, 4048, 4214, 3934, 4054, 4118, 4198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32684 payload_rows_remote=28498 dup_rows=84 dup_rows_remote=76 dedup_ratio=0.002563 router_tokens_per_expert=[2069, 2022, 2088, 2030, 2056, 2044, 2031, 2044] payload_rows_per_dst=[4186, 4026, 4268, 4222, 4090, 4110, 4066, 3716] dup_rows_per_dst=[8, 2, 18, 28, 0, 24, 4, 0] combine_rows_per_src=[4186, 4026, 4268, 4222, 4090, 4110, 4066, 3716] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=235225088 dispatch_meta_bytes=114856 combine_payload_bytes=235225088 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=235339944 ingress_bytes=2349154304 link_bytes=2349154304 token_bytes=470565032 weight_movement_bytes=2113929216 total_nvlink_bytes=2584494248 weight_to_token_ratio=4.492321 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315428352e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235438080 dispatch_meta_bytes=114960 combine_payload_bytes=235438080 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=235553040 ingress_bytes=3758653440 link_bytes=3758653440 token_bytes=470991120 weight_movement_bytes=3523215360 total_nvlink_bytes=3994206480 weight_to_token_ratio=7.480428 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.104689032e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28796 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2048, 2073, 2060, 2032, 2047, 2036, 1986, 2102] payload_rows_per_dst=[4096, 4146, 4120, 4064, 4094, 4072, 3972, 4204] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 4146, 4120, 4064, 4094, 4072, 3972, 4204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32724 payload_rows_remote=28456 dup_rows=44 dup_rows_remote=42 dedup_ratio=0.001343 router_tokens_per_expert=[2058, 2054, 2038, 1985, 2050, 2058, 2028, 2113] payload_rows_per_dst=[4022, 4106, 3906, 4196, 4042, 4128, 4056, 4268] dup_rows_per_dst=[0, 0, 0, 38, 0, 4, 0, 2] combine_rows_per_src=[4022, 4106, 3906, 4196, 4042, 4128, 4056, 4268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28806 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2063, 1998, 1981, 2123, 2038, 2097, 2120, 1964] payload_rows_per_dst=[4126, 3996, 3962, 4246, 4076, 4194, 4240, 3928] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4126, 3996, 3962, 4246, 4076, 4194, 4240, 3928] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234405888 dispatch_meta_bytes=114456 combine_payload_bytes=234405888 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=234520344 ingress_bytes=1643692032 link_bytes=1643692032 token_bytes=468926232 weight_movement_bytes=1409286144 total_nvlink_bytes=1878212376 weight_to_token_ratio=3.005347 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203989268e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=234635264 dispatch_meta_bytes=114568 combine_payload_bytes=234635264 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234749832 ingress_bytes=2348564480 link_bytes=2348564480 token_bytes=469385096 weight_movement_bytes=2113929216 total_nvlink_bytes=2583314312 weight_to_token_ratio=4.503614 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315098075e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233455616 dispatch_meta_bytes=114296 combine_payload_bytes=233455616 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=233569912 ingress_bytes=2347384832 link_bytes=2347384832 token_bytes=467025528 weight_movement_bytes=2113929216 total_nvlink_bytes=2580954744 weight_to_token_ratio=4.526368 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314437521e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=235896832 dispatch_meta_bytes=115184 combine_payload_bytes=235896832 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=236012016 ingress_bytes=2349826048 link_bytes=2349826048 token_bytes=471908848 weight_movement_bytes=2113929216 total_nvlink_bytes=2585838064 weight_to_token_ratio=4.479529 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315804500e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233111552 dispatch_meta_bytes=113992 combine_payload_bytes=233111552 prefetch_bytes=704643072 reduce_bytes=2113929216 egress_bytes=233225544 ingress_bytes=3051683840 link_bytes=3051683840 token_bytes=466337096 weight_movement_bytes=2818572288 total_nvlink_bytes=3284909384 weight_to_token_ratio=6.044066 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.708815567e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235978752 dispatch_meta_bytes=115224 combine_payload_bytes=235978752 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=236093976 ingress_bytes=2349907968 link_bytes=2349907968 token_bytes=472072728 weight_movement_bytes=2113929216 total_nvlink_bytes=2586001944 weight_to_token_ratio=4.477974 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315850372e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28630 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2037, 2058, 2053, 2060, 2058, 2069, 2013, 2036] payload_rows_per_dst=[4074, 4116, 4106, 4120, 4116, 4138, 4026, 4072] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4074, 4116, 4106, 4120, 4116, 4138, 4026, 4072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28584 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2132, 2056, 2028, 2044, 2092, 1885, 2108, 2039] payload_rows_per_dst=[4264, 4112, 4056, 4088, 4184, 3770, 4216, 4078] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4264, 4112, 4056, 4088, 4184, 3770, 4216, 4078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28888 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2048, 2029, 2036, 1940, 2082, 2054, 2067, 2128] payload_rows_per_dst=[4096, 4058, 4072, 3880, 4164, 4108, 4134, 4256] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4096, 4058, 4072, 3880, 4164, 4108, 4134, 4256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7521731464 prefetch_bytes=19730006016 reduce_bytes=15502147584 weight_movement_bytes=35232153600 egress_bytes=3761783688 ingress_bytes=38992101376 link_bytes=38992101376 total_nvlink_bytes=42753885064 weight_to_token_ratio=4.684048 prefetch_share_of_ingress=0.506000 dedup_ratio=0.000000 prefetch_slots=56 unused_prefetch_slots=8 slot_utilization=0.875000 reduce_slots=22 remote_demand=28 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017045 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.364621823e-04 prefetch_bytes_per_gemm_flop=6.904987374e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=234536960 dispatch_meta_bytes=114520 combine_payload_bytes=234536960 prefetch_bytes=1409286144 reduce_bytes=704643072 egress_bytes=234651480 ingress_bytes=2348466176 link_bytes=2348466176 token_bytes=469188440 weight_movement_bytes=2113929216 total_nvlink_bytes=2583117656 weight_to_token_ratio=4.505501 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315043029e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28522 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2024, 2123, 1985, 2079, 2059, 2012, 2052, 2050] payload_rows_per_dst=[4048, 4246, 3970, 4158, 4118, 4024, 4104, 4100] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4048, 4246, 3970, 4158, 4118, 4024, 4104, 4100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32654 payload_rows_remote=28434 dup_rows=114 dup_rows_remote=70 dedup_ratio=0.003479 router_tokens_per_expert=[2001, 1989, 2059, 2034, 1994, 2037, 2148, 2122] payload_rows_per_dst=[4220, 4044, 4170, 4062, 4066, 4150, 3708, 4234] dup_rows_per_dst=[44, 24, 4, 0, 36, 6, 0, 0] combine_rows_per_src=[4220, 4044, 4170, 4062, 4066, 4150, 3708, 4234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234160128 dispatch_meta_bytes=114336 combine_payload_bytes=234160128 prefetch_bytes=1409286144 reduce_bytes=2113929216 egress_bytes=234274464 ingress_bytes=3757375488 link_bytes=3757375488 token_bytes=468434592 weight_movement_bytes=3523215360 total_nvlink_bytes=3991649952 weight_to_token_ratio=7.521254 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.103973433e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28620 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2026, 2012, 2109, 2059, 2094, 2016, 2074, 1994] payload_rows_per_dst=[4052, 4024, 4218, 4118, 4188, 4032, 4148, 3988] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4052, 4024, 4218, 4118, 4188, 4032, 4148, 3988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32702 payload_rows_remote=28538 dup_rows=66 dup_rows_remote=52 dedup_ratio=0.002014 router_tokens_per_expert=[1989, 2081, 2078, 2027, 2112, 2026, 2033, 2038] payload_rows_per_dst=[3978, 4096, 4128, 4110, 3870, 4340, 4016, 4164] dup_rows_per_dst=[0, 0, 0, 10, 0, 42, 0, 14] combine_rows_per_src=[3978, 4096, 4128, 4110, 3870, 4340, 4016, 4164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=236650496 dispatch_meta_bytes=115552 combine_payload_bytes=236650496 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=236766048 ingress_bytes=1645936640 link_bytes=1645936640 token_bytes=473416544 weight_movement_bytes=1409286144 total_nvlink_bytes=1882702688 weight_to_token_ratio=2.976842 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.216558136e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=233652224 dispatch_meta_bytes=114088 combine_payload_bytes=233652224 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233766312 ingress_bytes=2347581440 link_bytes=2347581440 token_bytes=467418536 weight_movement_bytes=2113929216 total_nvlink_bytes=2581347752 weight_to_token_ratio=4.522562 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314547614e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7521928120 prefetch_bytes=18320719872 reduce_bytes=22548578304 weight_movement_bytes=40869298176 egress_bytes=3761882040 ingress_bytes=44629344256 link_bytes=44629344256 total_nvlink_bytes=48391226296 weight_to_token_ratio=5.433354 prefetch_share_of_ingress=0.410508 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=32 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018466 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.561910616e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=232931328 dispatch_meta_bytes=114016 combine_payload_bytes=232931328 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=233045344 ingress_bytes=1642217472 link_bytes=1642217472 token_bytes=465976672 weight_movement_bytes=1409286144 total_nvlink_bytes=1875262816 weight_to_token_ratio=3.024371 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195732348e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234455040 dispatch_meta_bytes=114480 combine_payload_bytes=234455040 prefetch_bytes=1409286144 reduce_bytes=3523215360 egress_bytes=234569520 ingress_bytes=5166956544 link_bytes=5166956544 token_bytes=469024560 weight_movement_bytes=4932501504 total_nvlink_bytes=5401526064 weight_to_token_ratio=10.516510 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=2.893279985e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=233783296 dispatch_meta_bytes=114360 combine_payload_bytes=233783296 prefetch_bytes=704643072 reduce_bytes=1409286144 egress_bytes=233897656 ingress_bytes=2347712512 link_bytes=2347712512 token_bytes=467680952 weight_movement_bytes=2113929216 total_nvlink_bytes=2581610168 weight_to_token_ratio=4.520024 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314621008e-04 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28738 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2039, 2071, 2010, 2030, 2096, 2015, 2080, 2043] payload_rows_per_dst=[4078, 4142, 4020, 4060, 4192, 4030, 4160, 4086] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4078, 4142, 4020, 4060, 4192, 4030, 4160, 4086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7467393040 prefetch_bytes=16911433728 reduce_bytes=28890365952 weight_movement_bytes=45801799680 egress_bytes=3734609936 ingress_bytes=49534582784 link_bytes=49534582784 total_nvlink_bytes=53269192720 weight_to_token_ratio=6.133573 prefetch_share_of_ingress=0.341407 dedup_ratio=0.002495 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=41 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.733581168e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7531564264 prefetch_bytes=21843935232 reduce_bytes=17616076800 weight_movement_bytes=39460012032 egress_bytes=3766701288 ingress_bytes=43224875008 link_bytes=43224875008 total_nvlink_bytes=46991576296 weight_to_token_ratio=5.239285 prefetch_share_of_ingress=0.505356 dedup_ratio=0.000000 prefetch_slots=62 unused_prefetch_slots=2 slot_utilization=0.968750 reduce_slots=25 remote_demand=31 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.015625 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.512757857e-04 prefetch_bytes_per_gemm_flop=7.644807449e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7461296656 prefetch_bytes=16911433728 reduce_bytes=25367150592 weight_movement_bytes=42278584320 egress_bytes=3731560976 ingress_bytes=46008320000 link_bytes=46008320000 total_nvlink_bytes=49739880976 weight_to_token_ratio=5.666386 prefetch_share_of_ingress=0.367573 dedup_ratio=0.002571 prefetch_slots=48 unused_prefetch_slots=16 slot_utilization=0.750000 reduce_slots=36 remote_demand=24 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.610171170e-04 prefetch_bytes_per_gemm_flop=5.918560606e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28728 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[2010, 2106, 2013, 2020, 2035, 2045, 2078, 2077] payload_rows_per_dst=[4020, 4212, 4026, 4040, 4070, 4090, 4156, 4154] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[4020, 4212, 4026, 4040, 4070, 4090, 4156, 4154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 E=8 B=1 K=2 R=8 NvS=8446 hidden_size=4096 assignments=16384 payload_rows=32768 payload_rows_remote=28650 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[1993, 2059, 2092, 2017, 2043, 2057, 2062, 2061] payload_rows_per_dst=[3986, 4118, 4184, 4034, 4086, 4114, 4124, 4122] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[3986, 4118, 4184, 4034, 4086, 4114, 4124, 4122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235421696 dispatch_meta_bytes=114952 combine_payload_bytes=235421696 prefetch_bytes=1409286144 reduce_bytes=0 egress_bytes=235536648 ingress_bytes=1644707840 link_bytes=1644707840 token_bytes=470958344 weight_movement_bytes=1409286144 total_nvlink_bytes=1880244488 weight_to_token_ratio=2.992380 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209677369e-05 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=2 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.500000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=0 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=1.000000 weight_bytes_per_expert=352321536 grad_bytes_per_expert=704643072 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=235339776 dispatch_meta_bytes=114912 combine_payload_bytes=235339776 prefetch_bytes=704643072 reduce_bytes=704643072 egress_bytes=235454688 ingress_bytes=1644625920 link_bytes=1644625920 token_bytes=470794464 weight_movement_bytes=1409286144 total_nvlink_bytes=1880080608 weight_to_token_ratio=2.993421 gemm_rows_defined=16512 gemm_rows_charged=16896 gemm_tail_pad_rows=384 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.022727 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.209218651e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=234700800 dispatch_meta_bytes=114600 combine_payload_bytes=234700800 prefetch_bytes=1409286144 reduce_bytes=1409286144 egress_bytes=234815400 ingress_bytes=3053273088 link_bytes=3053273088 token_bytes=469516200 weight_movement_bytes=2818572288 total_nvlink_bytes=3288088488 weight_to_token_ratio=6.003142 gemm_rows_defined=16640 gemm_rows_charged=16896 gemm_tail_pad_rows=256 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.015152 gemm_active_groups=4 gemm_peak_group_rows=8192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.709705479e-04 prefetch_bytes_per_gemm_flop=7.891414141e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7523271936 prefetch_bytes=21139292160 reduce_bytes=11978932224 weight_movement_bytes=33118224384 egress_bytes=3762554112 ingress_bytes=36878942208 link_bytes=36878942208 total_nvlink_bytes=40641496320 weight_to_token_ratio=4.402104 prefetch_share_of_ingress=0.573208 dedup_ratio=0.000000 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=17 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016098 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.290666765e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7528122784 prefetch_bytes=18320719872 reduce_bytes=16911433728 weight_movement_bytes=35232153600 egress_bytes=3764980128 ingress_bytes=38995296256 link_bytes=38995296256 total_nvlink_bytes=42760276384 weight_to_token_ratio=4.680072 prefetch_share_of_ingress=0.469819 dedup_ratio=0.000000 prefetch_slots=52 unused_prefetch_slots=12 slot_utilization=0.812500 reduce_slots=24 remote_demand=26 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.017992 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.364733635e-04 prefetch_bytes_per_gemm_flop=6.411773990e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=7514225760 prefetch_bytes=21139292160 reduce_bytes=15502147584 weight_movement_bytes=36641439744 egress_bytes=3758029920 ingress_bytes=40397635584 link_bytes=40397635584 total_nvlink_bytes=44155665504 weight_to_token_ratio=4.876276 prefetch_share_of_ingress=0.523280 dedup_ratio=0.000000 prefetch_slots=60 unused_prefetch_slots=4 slot_utilization=0.937500 reduce_slots=22 remote_demand=30 gemm_rows_charged=270336 gemm_tail_pad_fraction=0.016098 gemm_inner_pad_upper_bound=8128 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.413811853e-04 prefetch_bytes_per_gemm_flop=7.398200758e-05 anomalies=none - [2026-09-16 21:35:56] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 573.1 | throughput per GPU (TFLOP/s/GPU): 581.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.435178E+00 | loss scale: 1.0 | grad norm: 3.374 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-09-16 21:35:56 -[rank0]:[W916 21:35:58.261944386 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W916 21:35:59.304429309 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W916 21:35:59.315228175 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W916 21:35:59.320716830 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W916 21:35:59.326075658 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W916 21:35:59.510533913 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W916 21:35:59.535389820 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W916 21:35:59.780941048 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0916 21:36:05.699000 31788 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0916 21:36:05.699000 31788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 31978 closing signal SIGTERM -Traceback (most recent call last): - File "/usr/local/bin/torchrun", line 8, in - sys.exit(main()) - ^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper - return f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main - run(args) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run - elastic_launch( - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent - result = agent.run() - ^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper - result = f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run - result = self._invoke_run(role) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run - time.sleep(monitor_interval) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 31788 got signal: 15 diff --git a/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log b/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log index f281d17cfbc01ae2bfa25d87f6c58e2f533a3344..142babaf78768e65141d359645a60c285c455677 100644 --- a/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log +++ b/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log @@ -1,4959 +1,3 @@ -[21:36:06] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -============================================================== - G2 / moonep (slots/rank=4 policy=matched) - E=32 K=8 H'=3584 m=4 - L=16 seq=4096 gbs=16 (2 microbatches) - cap_mode=none cap=none cg=0 recompute=none - memory_dump=none snapshots=/workspace/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ a6a1a221 -============================================================== -W0916 21:36:15.933000 34463 torch/distributed/run.py:766] -W0916 21:36:15.933000 34463 torch/distributed/run.py:766] ***************************************** -W0916 21:36:15.933000 34463 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0916 21:36:15.933000 34463 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.auto - attention_dropout ............................... 0.0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_scope ................................ full - cuda_graph_warmup_steps ......................... 3 - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 8 - data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_type ................................. single - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - delay_wgrad_compute ............................. False - deprecated_use_mcore_models ..................... True - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_bf16_reduced_precision_matmul ........... False - disable_mamba_mem_eff_path ...................... False - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - embedding_init_method_std ....................... None - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_experimental ............................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_msc ...................................... True - enable_one_logger ............................... True - encoder_num_layers .............................. 16 - encoder_seq_length .............................. 4096 - end_weight_decay ................................ 0.1 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000000 - eval_iters ...................................... 0 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 8 - expert_tensor_parallel_size ..................... 1 - export_force_local_attention .................... False - export_kd_cfg ................................... None - export_kd_teacher_ckpt_format ................... None - export_kd_teacher_load .......................... None - export_kv_cache_quant ........................... False - export_legacy_megatron .......................... False - export_model_type ............................... GPTModel - export_moe_apply_probs_on_input ................. False - export_qk_l2_norm ............................... False - export_quant_cfg ................................ None - export_real_quant_cfg ........................... None - export_te_mcore_model ........................... False - external_cuda_graph ............................. False - ffn_hidden_size ................................. 14336 - finetune ........................................ False - finetune_data_split ............................. train - finetune_hf_dataset ............................. None - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. e4m3 - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... mxfp8 - fp8_wgrad ....................................... True - fsdp_double_buffer .............................. False - global_batch_size ............................... 16 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... True - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - heterogeneous_layers_config_encoded_json ........ None - heterogeneous_layers_config_path ................ None - hidden_dropout .................................. 0.0 - hidden_size ..................................... 4096 - hierarchical_context_parallel_sizes ............. None - high_priority_stream_groups ..................... [] - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_dynamic_batching ...................... False - inference_dynamic_batching_buffer_guaranteed_fraction 0.2 - inference_dynamic_batching_buffer_overflow_factor None - inference_dynamic_batching_buffer_size_gb ....... 40.0 - inference_dynamic_batching_chunk_size ........... 256 - inference_dynamic_batching_max_requests_override None - inference_dynamic_batching_max_tokens_override .. None - inference_dynamic_batching_num_cuda_graphs ...... 16 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.01 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 4294967296 - inprocess_active_world_size ..................... 8 - inprocess_barrier_timeout ....................... 120 - inprocess_completion_timeout .................... 120 - inprocess_empty_cuda_cache ...................... False - inprocess_granularity ........................... node - inprocess_hard_timeout .......................... 90 - inprocess_heartbeat_interval .................... 30 - inprocess_heartbeat_timeout ..................... 60 - inprocess_last_call_wait ........................ 1 - inprocess_max_iterations ........................ None - inprocess_monitor_process_interval .............. 1.0 - inprocess_monitor_thread_interval ............... 1.0 - inprocess_progress_watchdog_interval ............ 1.0 - inprocess_restart ............................... False - inprocess_soft_timeout .......................... 60 - inprocess_termination_grace_time ................ 1 - is_hybrid_model ................................. False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kitchen_config_file ............................. None - kitchen_recipe_number ........................... None - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - lazy_mpu_init ................................... None - load ............................................ None - load_main_params_from_ckpt ...................... None - load_model_opt_format ........................... False - local_rank ...................................... 0 - log_energy ...................................... False - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. True - log_timers_to_tensorboard ....................... False - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 8 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. None - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 5 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - mamba_head_dim .................................. 64 - mamba_num_groups ................................ 8 - mamba_num_heads ................................. None - mamba_state_dim ................................. 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... False - max_position_embeddings ......................... 32768 - max_tokens_to_oom ............................... 12000 - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - mid_level_dataset_surplus ....................... 0.005 - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-05 - mlp_chunks_for_prefill .......................... 1 - mmap_bin_files .................................. True - mock_data ....................................... False - moe_apply_probs_on_input ........................ False - moe_aux_loss_coeff .............................. 0.01 - moe_deepep_num_sms .............................. 20 - moe_echo_enable_random_offloading ............... False - moe_echo_expert_dispatch_overlap ................ False - moe_echo_expert_dispatcher_type ................. hybridep - moe_echo_recompute_expert_dispatch .............. False - moe_enable_deepep ............................... False - moe_enable_echo ................................. False - moe_enable_hybridep ............................. False - moe_enable_moonep ............................... True - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. 3584 - moe_grouped_gemm ................................ True - moe_hybridep_num_sms ............................ 16 - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_moonep_async_grad_reduce .................... True - moe_moonep_num_sms .............................. None - moe_moonep_traffic_logging ...................... True - moe_moonep_traffic_logging_interval ............. 1 - moe_moonep_traffic_warmup_iters ................. 2 - moe_num_echo_experts ............................ None - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_received_token_capacity ..................... None - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ fp32 - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 8 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... flex - moe_token_drop_policy ........................... probs - moe_upcycling_granularity ....................... 1 - moe_use_device_initiated_grouped_gemm ........... True - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mrope_section ................................... None - mscale .......................................... 1.0 - mscale_all_dim .................................. 0.0 - mtp_loss_scaling_factor ......................... 0.1 - mtp_num_layers .................................. None - multi_latent_attention .......................... False - nccl_all_reduce_for_prefill ..................... False - nccl_communicator_config_path ................... None - nccl_ub ......................................... False - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_rope_freq .................................... None - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-05 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... 32 - num_layers ...................................... 16 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 2 - object_storage_cache_path ....................... None - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_grad_reduce ............................. True - overlap_moe_expert_parallel_comm ................ False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ True - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_layout .................. None - pipeline_model_parallel_size .................... 1 - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_l2_norm ...................................... False - qk_layernorm .................................... False - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... None - recompute_method ................................ None - recompute_modules ............................... None - recompute_num_layers ............................ None - record_memory_history ........................... False - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... validate_results - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - reuse_grad_buf_for_mxfp8_param_ag ............... False - rope_scaling_factor ............................. 8.0 - rope_type ....................................... None - rotary_base ..................................... 1000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - run_workload_inspector_server ................... False - sample_rate ..................................... 1.0 - save ............................................ None - save_interval ................................... None - save_retain_interval ............................ None - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - seq_length ...................................... 4096 - sequence_parallel ............................... False - sft ............................................. False - sft_tokenizer_prompt_format ..................... nemotron-h-aligned - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - squared_relu .................................... False - start_weight_decay .............................. 0.1 - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - suggested_communication_unit_size ............... None - swiglu .......................................... True - swin_backbone_type .............................. tiny - symmetric_ar_type ............................... None - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. None - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. GPT2BPETokenizer - torch_fsdp2_reshard_after_forward ............... True - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 12 - train_samples ................................... None - train_sync_interval ............................. None - transformer_impl ................................ transformer_engine - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... True - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_sharp ....................................... False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ False - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json - vocab_size ...................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.1 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 8 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> building GPT2BPETokenizer tokenizer ... - > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS -> initializing torch distributed ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank -7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 37456 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 -777 - - - -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0[Gloo] Rank is connected to 00 peer ranks. is connected to Expected number of connected peer ranks is : 00 peer ranks. -Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank7]:[W916 21:36:32.197412874 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W916 21:36:32.198696315 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W916 21:36:32.198721174 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W916 21:36:32.198782450 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W916 21:36:32.198813879 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W916 21:36:32.198836700 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W916 21:36:32.198853039 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -> compiling dataset index builder ... -make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.034 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -[rank0]:[W916 21:36:32.356091978 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 4.043 seconds -time to initialize megatron (seconds): 7.291 -[after megatron is initialized] datetime: 2026-09-16 21:36:38 -building GPT model ... -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3903983616 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 -Params for bucket 1 (206045184 elements, 206045184 padded size): - module.output_layer.weight -Params for bucket 2 (42086400 elements, 42086400 padded size): - module.decoder.final_layernorm.weight - module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.layers.15.self_attention.linear_qkv.weight - module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.mlp.router.weight - module.decoder.layers.15.self_attention.linear_proj.weight -Params for bucket 3 (42082304 elements, 42082304 padded size): - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.self_attention.linear_qkv.weight - module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.14.pre_mlp_layernorm.weight -Params for bucket 4 (42082304 elements, 42082304 padded size): - module.decoder.layers.13.pre_mlp_layernorm.weight - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.13.self_attention.linear_qkv.weight -Params for bucket 5 (42082304 elements, 42082304 padded size): - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.self_attention.linear_qkv.weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.12.pre_mlp_layernorm.weight -Params for bucket 6 (42082304 elements, 42082304 padded size): - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.11.self_attention.linear_qkv.weight - module.decoder.layers.11.self_attention.linear_proj.weight -Params for bucket 7 (42082304 elements, 42082304 padded size): - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.pre_mlp_layernorm.weight -Params for bucket 8 (42082304 elements, 42082304 padded size): - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.self_attention.linear_qkv.weight - module.decoder.layers.9.self_attention.linear_proj.weight -Params for bucket 9 (42082304 elements, 42082304 padded size): - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.8.pre_mlp_layernorm.weight -Params for bucket 10 (42082304 elements, 42082304 padded size): - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.7.pre_mlp_layernorm.weight -Params for bucket 11 (42082304 elements, 42082304 padded size): - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.6.self_attention.linear_proj.weight -Params for bucket 12 (42082304 elements, 42082304 padded size): - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.5.pre_mlp_layernorm.weight -Params for bucket 13 (42082304 elements, 42082304 padded size): - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.pre_mlp_layernorm.weight -Params for bucket 14 (42082304 elements, 42082304 padded size): - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight -Params for bucket 15 (42082304 elements, 42082304 padded size): - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.2.pre_mlp_layernorm.weight -Params for bucket 16 (42082304 elements, 42082304 padded size): - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_qkv.weight -Params for bucket 17 (42082304 elements, 42082304 padded size): - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.linear_proj.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.pre_mlp_layernorm.weight -Params for bucket 18 (206045184 elements, 206045184 padded size): - module.embedding.word_embeddings.weight -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 56 -Params for bucket 1 (44040192 elements, 44040192 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight0 - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 -Params for bucket 2 (44040192 elements, 44040192 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 -Params for bucket 3 (58720256 elements, 58720256 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight1 - module.decoder.layers.15.mlp.experts.linear_fc1.weight2 -Params for bucket 4 (44040192 elements, 44040192 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc2.weight0 -Params for bucket 5 (44040192 elements, 44040192 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight1 - module.decoder.layers.14.mlp.experts.linear_fc2.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight3 -Params for bucket 6 (58720256 elements, 58720256 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 - module.decoder.layers.14.mlp.experts.linear_fc1.weight1 -Params for bucket 7 (58720256 elements, 58720256 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight2 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 -Params for bucket 8 (44040192 elements, 44040192 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight1 - module.decoder.layers.13.mlp.experts.linear_fc2.weight2 -Params for bucket 9 (44040192 elements, 44040192 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight3 - module.decoder.layers.13.mlp.experts.linear_fc1.weight0 -Params for bucket 10 (58720256 elements, 58720256 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight1 - module.decoder.layers.13.mlp.experts.linear_fc1.weight2 -Params for bucket 11 (44040192 elements, 44040192 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight3 - module.decoder.layers.12.mlp.experts.linear_fc2.weight0 -Params for bucket 12 (44040192 elements, 44040192 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight3 - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 -Params for bucket 13 (58720256 elements, 58720256 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 -Params for bucket 14 (58720256 elements, 58720256 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 - module.decoder.layers.12.mlp.experts.linear_fc1.weight3 -Params for bucket 15 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight2 -Params for bucket 16 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight3 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 -Params for bucket 17 (58720256 elements, 58720256 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 -Params for bucket 18 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 -Params for bucket 19 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 -Params for bucket 20 (58720256 elements, 58720256 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 -Params for bucket 21 (58720256 elements, 58720256 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight3 -Params for bucket 22 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight2 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 -Params for bucket 23 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 -Params for bucket 24 (58720256 elements, 58720256 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 -Params for bucket 25 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 -Params for bucket 26 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 -Params for bucket 27 (58720256 elements, 58720256 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 -Params for bucket 28 (58720256 elements, 58720256 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 -Params for bucket 29 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 -Params for bucket 30 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 -Params for bucket 31 (58720256 elements, 58720256 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight2 -Params for bucket 32 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 -Params for bucket 33 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight3 -Params for bucket 34 (58720256 elements, 58720256 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 -Params for bucket 35 (58720256 elements, 58720256 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 -Params for bucket 36 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 -Params for bucket 37 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 -Params for bucket 38 (58720256 elements, 58720256 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 -Params for bucket 39 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 -Params for bucket 40 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 -Params for bucket 41 (58720256 elements, 58720256 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 -Params for bucket 42 (58720256 elements, 58720256 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 -Params for bucket 43 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 -Params for bucket 44 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 -Params for bucket 45 (58720256 elements, 58720256 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 -Params for bucket 46 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 -Params for bucket 47 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 -Params for bucket 48 (58720256 elements, 58720256 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 -Params for bucket 49 (58720256 elements, 58720256 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 - module.decoder.layers.2.mlp.experts.linear_fc1.weight3 -Params for bucket 50 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 -Params for bucket 51 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 -Params for bucket 52 (58720256 elements, 58720256 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc1.weight2 -Params for bucket 53 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 -Params for bucket 54 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 -Params for bucket 55 (58720256 elements, 58720256 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc1.weight1 -Params for bucket 56 (58720256 elements, 58720256 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight3 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-16 21:36:41 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 192 - validation: 0 - test: 0 -INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] -> building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) -INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers -INFO:megatron.core.datasets.indexed_dataset: Extract the document indices -INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 -INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 -INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices -INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy -INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy -INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy -INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 -> finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-09-16 21:36:41 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3111.71, 3114.46) - train/valid/test-data-iterators-setup ..........: (67.04, 72.05) -training ... -Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-09-16 21:36:41 -INFO:moonep.api:MoonEP comm buffer sizes: - config : R=8, S=4096, H=4096, E=32, K=8, NvS=33784, NvS_padded=33792 - sm_count : num_sms=32, num_sms_dedup=148 - hidden_buffer_size : 264.00 MiB - meta_buffer_size : 2.00 MiB - local_temp_buffer_size : 1.25 MiB - total_size : 267.25 MiB -[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 32, 33, 34, 35] sizes [1152, 21760, 6784, 3200, 128, 0, 0, 0] | duplicated experts [[4, -1, -1, -1], [-1, -1, -1, -1], [23, -1, -1, -1], [22, -1, -1, -1], [31, -1, -1, -1], [30, -1, -1, -1], [1, -1, -1, -1], [4, -1, -1, -1]] -[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 32, 33, 34, 35] sizes [6528, 128, 11392, 896, 14208, 0, 0, 0] | duplicated experts [[4, -1, -1, -1], [28, -1, -1, -1], [19, -1, -1, -1], [28, -1, -1, -1], [13, -1, -1, -1], [27, -1, -1, -1], [7, -1, -1, -1], [-1, -1, -1, -1]] -[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 32, 33, 34, 35] sizes [9984, 11264, 768, 11136, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1], [23, -1, -1, -1], [3, -1, -1, -1], [3, -1, -1, -1], [0, -1, -1, -1], [13, -1, -1, -1], [17, -1, -1, -1], [6, -1, -1, -1]] -[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 32, 33, 34, 35] sizes [13184, 4480, 384, 14208, 768, 0, 0, 0] | duplicated experts [[19, -1, -1, -1], [12, -1, -1, -1], [17, -1, -1, -1], [0, -1, -1, -1], [-1, -1, -1, -1], [0, -1, -1, -1], [12, -1, -1, -1], [3, -1, -1, -1]] -WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-09-16 21:37:03] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 21874.1 | throughput per GPU (TFLOP/s/GPU): 15.2 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102610E+01 | loss scale: 1.0 | grad norm: 44.466 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 23.22 -Number of parameters in embedding layers in billions: 0.41 -Total number of parameters in billions: 23.63 -Number of parameters in most loaded shard in billions: 23.6320 -Theoretical memory footprints: weight and optimizer=169029.43 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 53053.056640625 | max allocated: 53053.0576171875 | reserved: 56290.0 | max reserved: 56290.0 - [2026-09-16 21:37:04] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 612.4 | throughput per GPU (TFLOP/s/GPU): 544.3 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101605E+01 | loss scale: 1.0 | grad norm: 45.876 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=93680 payload_rows_remote=80112 dup_rows=37392 dup_rows_remote=33022 dedup_ratio=0.285278 router_tokens_per_expert=[235, 6160, 1836, 369, 3238, 298, 1816, 854, 2198, 382, 3613, 1702, 1677, 1692, 279, 1118, 768, 1371, 3399, 3798, 4707, 367, 5175, 2481, 120, 3241, 1228, 2802, 2069, 819, 2892, 2832] payload_rows_per_dst=[13568, 12374, 12816, 13110, 11252, 11730, 11294, 7536] dup_rows_per_dst=[4370, 5566, 6052, 6772, 4908, 4990, 3488, 1246] combine_rows_per_src=[13568, 12374, 12816, 13110, 11252, 11730, 11294, 7536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97518 payload_rows_remote=83800 dup_rows=33554 dup_rows_remote=26608 dedup_ratio=0.255997 router_tokens_per_expert=[268, 5849, 1374, 229, 5320, 275, 1823, 1097, 2385, 184, 4204, 1090, 1742, 1762, 425, 826, 478, 1417, 2752, 4152, 3726, 767, 5577, 2695, 168, 2657, 1116, 2373, 1675, 752, 3761, 2617] payload_rows_per_dst=[12604, 12712, 12104, 13718, 12868, 11066, 10052, 12394] dup_rows_per_dst=[2836, 4318, 3622, 6946, 4730, 3310, 2576, 5216] combine_rows_per_src=[12604, 12712, 12104, 13718, 12868, 11066, 10052, 12394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94300 payload_rows_remote=81550 dup_rows=36772 dup_rows_remote=29048 dedup_ratio=0.280548 router_tokens_per_expert=[120, 6223, 2198, 287, 4070, 168, 1827, 1078, 2811, 275, 3731, 865, 1189, 1668, 231, 785, 464, 1161, 3194, 4228, 3799, 869, 4834, 2643, 165, 3065, 1161, 2331, 2662, 760, 3910, 2764] payload_rows_per_dst=[13338, 10876, 11912, 6802, 11636, 13880, 12750, 13106] dup_rows_per_dst=[4318, 3410, 3452, 1008, 2920, 6854, 7724, 7086] combine_rows_per_src=[13338, 10876, 11912, 6802, 11636, 13880, 12750, 13106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=93380 payload_rows_remote=81828 dup_rows=37692 dup_rows_remote=32690 dedup_ratio=0.287567 router_tokens_per_expert=[212, 5894, 1305, 285, 5064, 256, 1534, 1315, 2758, 194, 3853, 1579, 1355, 1595, 354, 957, 879, 1831, 3469, 4815, 3696, 503, 6174, 2109, 141, 2344, 1041, 2008, 1711, 592, 3116, 2597] payload_rows_per_dst=[10262, 12238, 13072, 7262, 11904, 14380, 12710, 11552] dup_rows_per_dst=[1794, 4100, 6516, 1260, 5078, 7116, 6826, 5002] combine_rows_per_src=[10262, 12238, 13072, 7262, 11904, 14380, 12710, 11552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95198 payload_rows_remote=82354 dup_rows=35874 dup_rows_remote=28660 dedup_ratio=0.273697 router_tokens_per_expert=[197, 6149, 1654, 453, 4002, 263, 2206, 1196, 2696, 329, 4071, 863, 1955, 1825, 433, 1153, 485, 1289, 3330, 3506, 3961, 871, 4206, 2542, 303, 2618, 1460, 2101, 2243, 776, 3632, 2768] payload_rows_per_dst=[13174, 12844, 12232, 12614, 10868, 11128, 10420, 11918] dup_rows_per_dst=[3732, 7214, 3686, 6530, 3526, 3620, 2544, 5022] combine_rows_per_src=[13174, 12844, 12232, 12614, 10868, 11128, 10420, 11918] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=656277504 dispatch_meta_bytes=452536 combine_payload_bytes=656277504 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=656730040 ingress_bytes=1360920576 link_bytes=1360920576 token_bytes=1313007544 weight_movement_bytes=704643072 total_nvlink_bytes=2017650616 weight_to_token_ratio=0.536663 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=24064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.620587149e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94412 payload_rows_remote=82202 dup_rows=36660 dup_rows_remote=32940 dedup_ratio=0.279694 router_tokens_per_expert=[106, 6094, 2446, 176, 4573, 291, 2162, 1136, 2084, 215, 3287, 1291, 1636, 1606, 421, 957, 491, 1694, 2612, 4325, 3718, 706, 5561, 2473, 199, 3480, 1670, 2623, 1824, 702, 2261, 2716] payload_rows_per_dst=[13196, 11868, 10704, 11002, 10934, 12210, 13606, 10892] dup_rows_per_dst=[4448, 4456, 3050, 3546, 3034, 3720, 10292, 4114] combine_rows_per_src=[13196, 11868, 10704, 11002, 10934, 12210, 13606, 10892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95600 payload_rows_remote=84190 dup_rows=35472 dup_rows_remote=32386 dedup_ratio=0.270630 router_tokens_per_expert=[254, 6535, 2238, 324, 4071, 246, 1664, 1152, 2606, 237, 3643, 762, 2015, 1742, 233, 954, 746, 1816, 2669, 4120, 3946, 400, 5279, 2216, 144, 2332, 1610, 2978, 2387, 637, 2630, 2950] payload_rows_per_dst=[13888, 10760, 11410, 13346, 13214, 10318, 10876, 11788] dup_rows_per_dst=[4814, 3506, 3086, 7100, 5488, 2806, 3252, 5420] combine_rows_per_src=[13888, 10760, 11410, 13346, 13214, 10318, 10876, 11788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=686489600 dispatch_meta_bytes=441632 combine_payload_bytes=686489600 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=686931232 ingress_bytes=1038811136 link_bytes=1038811136 token_bytes=1373420832 weight_movement_bytes=352321536 total_nvlink_bytes=1725742368 weight_to_token_ratio=0.256528 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=14976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.816908740e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=668057600 dispatch_meta_bytes=442392 combine_payload_bytes=668057600 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=668499992 ingress_bytes=1020379136 link_bytes=1020379136 token_bytes=1336557592 weight_movement_bytes=352321536 total_nvlink_bytes=1688879128 weight_to_token_ratio=0.263604 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=11520 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.713697234e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=670334976 dispatch_meta_bytes=458072 combine_payload_bytes=670334976 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=670793048 ingress_bytes=1551138816 link_bytes=1551138816 token_bytes=1341128024 weight_movement_bytes=880803840 total_nvlink_bytes=2221931864 weight_to_token_ratio=0.656763 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=11392 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.685729892e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=674643968 dispatch_meta_bytes=444056 combine_payload_bytes=674643968 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=675088024 ingress_bytes=1026965504 link_bytes=1026965504 token_bytes=1349731992 weight_movement_bytes=352321536 total_nvlink_bytes=1702053528 weight_to_token_ratio=0.261031 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=17536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.750578146e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=673398784 dispatch_meta_bytes=460568 combine_payload_bytes=673398784 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=673859352 ingress_bytes=1730363392 link_bytes=1730363392 token_bytes=1347258136 weight_movement_bytes=1056964608 total_nvlink_bytes=2404222744 weight_to_token_ratio=0.784530 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=14848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.689312706e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=689684480 dispatch_meta_bytes=466304 combine_payload_bytes=689684480 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=690150784 ingress_bytes=1042006016 link_bytes=1042006016 token_bytes=1379835264 weight_movement_bytes=352321536 total_nvlink_bytes=1732156800 weight_to_token_ratio=0.255336 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=15872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.834798735e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107400 payload_rows_remote=91042 dup_rows=23672 dup_rows_remote=21916 dedup_ratio=0.180603 router_tokens_per_expert=[361, 101, 8158, 24, 272, 0, 5294, 97, 8, 0, 0, 54, 86, 0, 2, 51, 910, 3, 5, 30, 6113, 3, 6098, 83, 7872, 4906, 6907, 2881, 7998, 24, 1641, 5554] payload_rows_per_dst=[16358, 15268, 16346, 16268, 13862, 15832, 10268, 3198] dup_rows_per_dst=[1756, 7818, 2950, 908, 1664, 7284, 1292, 0] combine_rows_per_src=[16358, 15268, 16346, 16268, 13862, 15832, 10268, 3198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=745816064 dispatch_meta_bytes=451832 combine_payload_bytes=745816064 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=746267896 ingress_bytes=1450459136 link_bytes=1450459136 token_bytes=1492083960 weight_movement_bytes=704643072 total_nvlink_bytes=2196727032 weight_to_token_ratio=0.472254 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.024621 gemm_active_groups=11 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.121965710e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110922 payload_rows_remote=94674 dup_rows=20150 dup_rows_remote=20036 dedup_ratio=0.153732 router_tokens_per_expert=[14, 1556, 7715, 344, 1, 12, 3348, 70, 45, 17, 55, 41, 0, 303, 0, 16, 2269, 31, 6, 0, 4599, 11, 1542, 89, 7980, 7014, 7565, 3660, 8178, 0, 3222, 5833] payload_rows_per_dst=[15884, 15832, 16302, 16248, 15902, 5396, 15292, 10066] dup_rows_per_dst=[3374, 5032, 302, 114, 4144, 860, 6056, 268] combine_rows_per_src=[15884, 15832, 16302, 16248, 15902, 5396, 15292, 10066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108154 payload_rows_remote=97070 dup_rows=22918 dup_rows_remote=20664 dedup_ratio=0.174850 router_tokens_per_expert=[407, 2926, 7962, 4, 14, 0, 2807, 118, 2, 0, 10, 27, 3, 18, 3, 1, 557, 0, 11, 3, 5230, 30, 3421, 144, 7601, 4198, 8066, 1321, 8174, 5, 5690, 6783] payload_rows_per_dst=[13438, 5862, 15258, 16152, 16364, 14296, 11084, 15700] dup_rows_per_dst=[1070, 16, 16, 40, 1122, 2178, 2254, 16222] combine_rows_per_src=[13438, 5862, 15258, 16152, 16364, 14296, 11084, 15700] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97596 payload_rows_remote=81958 dup_rows=33476 dup_rows_remote=20452 dedup_ratio=0.255402 router_tokens_per_expert=[98, 737, 7128, 64, 2, 0, 5633, 247, 1, 0, 8, 0, 13, 0, 2, 9, 196, 4, 26, 1, 6322, 6, 2302, 29, 8125, 6991, 7902, 1037, 8151, 0, 4222, 6280] payload_rows_per_dst=[4888, 11734, 16122, 15862, 9040, 7958, 16354, 15638] dup_rows_per_dst=[708, 30, 10, 18, 218, 1780, 17688, 13024] combine_rows_per_src=[4888, 11734, 16122, 15862, 9040, 7958, 16354, 15638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112698 payload_rows_remote=99106 dup_rows=18374 dup_rows_remote=13956 dedup_ratio=0.140182 router_tokens_per_expert=[118, 1729, 7987, 555, 14, 0, 5291, 0, 12, 172, 13, 5, 24, 34, 4, 0, 390, 2, 11, 53, 5237, 4, 2618, 46, 7682, 4124, 8136, 3795, 7961, 3, 3676, 5840] payload_rows_per_dst=[16040, 13592, 16190, 16316, 15454, 10700, 11630, 12776] dup_rows_per_dst=[4738, 4418, 132, 114, 792, 1724, 4208, 2248] combine_rows_per_src=[16040, 13592, 16190, 16316, 15454, 10700, 11630, 12776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116750 payload_rows_remote=101778 dup_rows=14322 dup_rows_remote=8450 dedup_ratio=0.109268 router_tokens_per_expert=[499, 1733, 7850, 127, 4, 47, 4347, 270, 56, 1, 0, 24, 0, 1, 97, 30, 30, 10, 7, 0, 6269, 21, 4101, 0, 7494, 4021, 8041, 964, 8172, 98, 3954, 7268] payload_rows_per_dst=[12632, 15000, 15404, 16356, 15750, 14972, 11660, 14976] dup_rows_per_dst=[3204, 876, 94, 250, 72, 5872, 2830, 1124] combine_rows_per_src=[12632, 15000, 15404, 16356, 15750, 14972, 11660, 14976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114578 payload_rows_remote=98512 dup_rows=16494 dup_rows_remote=16490 dedup_ratio=0.125839 router_tokens_per_expert=[68, 1602, 7981, 76, 0, 0, 3616, 496, 7, 0, 7, 0, 0, 39, 13, 6, 644, 3, 3, 6, 3020, 6, 7640, 154, 8116, 3803, 8034, 1784, 8021, 1, 5559, 4831] payload_rows_per_dst=[16166, 13480, 16066, 15990, 16374, 15460, 8750, 12292] dup_rows_per_dst=[3288, 4582, 4, 62, 1302, 1162, 2424, 3670] combine_rows_per_src=[16166, 13480, 16066, 15990, 16374, 15460, 8750, 12292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=775569408 dispatch_meta_bytes=458840 combine_payload_bytes=775569408 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=776028248 ingress_bytes=1480212480 link_bytes=1480212480 token_bytes=1551597656 weight_movement_bytes=704643072 total_nvlink_bytes=2256240728 weight_to_token_ratio=0.454140 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.020833 gemm_active_groups=14 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.288572017e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=795197440 dispatch_meta_bytes=470936 combine_payload_bytes=795197440 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=795668376 ingress_bytes=2204483584 link_bytes=2204483584 token_bytes=1590865816 weight_movement_bytes=1409286144 total_nvlink_bytes=3000151960 weight_to_token_ratio=0.885861 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.234418787e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=671399936 dispatch_meta_bytes=409640 combine_payload_bytes=671399936 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=671809576 ingress_bytes=1904525312 link_bytes=1904525312 token_bytes=1343209512 weight_movement_bytes=1233125376 total_nvlink_bytes=2576334888 weight_to_token_ratio=0.918044 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.066454676e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105918 payload_rows_remote=89824 dup_rows=25154 dup_rows_remote=20228 dedup_ratio=0.191910 router_tokens_per_expert=[77, 7440, 18, 51, 220, 27, 2, 7354, 0, 8024, 0, 31, 513, 6, 8120, 27, 2, 26, 22, 5511, 2639, 8188, 0, 3288, 8191, 115, 4008, 563, 54, 805, 59, 155] payload_rows_per_dst=[16094, 13050, 16088, 9326, 11120, 16376, 9314, 14550] dup_rows_per_dst=[4926, 2268, 1158, 1144, 2, 15238, 58, 360] combine_rows_per_src=[16094, 13050, 16088, 9326, 11120, 16376, 9314, 14550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=811876352 dispatch_meta_bytes=452248 combine_payload_bytes=811876352 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=812328600 ingress_bytes=1164197888 link_bytes=1164197888 token_bytes=1624204952 weight_movement_bytes=352321536 total_nvlink_bytes=1976526488 weight_to_token_ratio=0.216919 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=20608 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.519022213e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=833765376 dispatch_meta_bytes=440912 combine_payload_bytes=833765376 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=834206288 ingress_bytes=1538408448 link_bytes=1538408448 token_bytes=1667971664 weight_movement_bytes=704643072 total_nvlink_bytes=2372614736 weight_to_token_ratio=0.422455 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=17536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.614445146e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=807010304 dispatch_meta_bytes=460008 combine_payload_bytes=807010304 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=807470312 ingress_bytes=1335492608 link_bytes=1335492608 token_bytes=1614480616 weight_movement_bytes=528482304 total_nvlink_bytes=2142962920 weight_to_token_ratio=0.327339 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.018939 gemm_active_groups=13 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.478201143e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=735838208 dispatch_meta_bytes=440208 combine_payload_bytes=735838208 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=736278416 ingress_bytes=1264320512 link_bytes=1264320512 token_bytes=1472116624 weight_movement_bytes=528482304 total_nvlink_bytes=2000598928 weight_to_token_ratio=0.358995 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.079667114e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105494 payload_rows_remote=89114 dup_rows=25578 dup_rows_remote=14854 dedup_ratio=0.195145 router_tokens_per_expert=[812, 8171, 2, 1, 1, 3021, 0, 7906, 2, 8146, 0, 3, 161, 3778, 7440, 0, 0, 26, 91, 7878, 338, 8185, 0, 31, 8190, 1, 1266, 0, 6, 38, 2, 40] payload_rows_per_dst=[16348, 15526, 16296, 8332, 15756, 1676, 16380, 15180] dup_rows_per_dst=[1624, 4862, 522, 4994, 2748, 14, 10724, 90] combine_rows_per_src=[16348, 15526, 16296, 8332, 15756, 1676, 16380, 15180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115790 payload_rows_remote=99962 dup_rows=15282 dup_rows_remote=4240 dedup_ratio=0.116592 router_tokens_per_expert=[659, 7764, 0, 0, 0, 465, 0, 8082, 0, 8155, 5, 6, 147, 5398, 7817, 0, 98, 83, 2, 7922, 36, 8186, 0, 268, 8167, 11, 1659, 11, 67, 371, 152, 5] payload_rows_per_dst=[15580, 16200, 16316, 15828, 16058, 14052, 5454, 16302] dup_rows_per_dst=[1266, 894, 16, 11042, 152, 536, 296, 1080] combine_rows_per_src=[15580, 16200, 16316, 15828, 16058, 14052, 5454, 16302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=106474 payload_rows_remote=90278 dup_rows=24598 dup_rows_remote=24222 dedup_ratio=0.187668 router_tokens_per_expert=[265, 8160, 0, 0, 1, 149, 192, 7731, 16, 8135, 0, 30, 69, 4048, 7862, 9, 0, 4, 3, 8028, 260, 8185, 0, 3718, 8162, 7, 198, 2, 0, 141, 48, 113] payload_rows_per_dst=[9436, 7842, 16332, 8212, 16176, 15896, 16384, 16196] dup_rows_per_dst=[514, 116, 30, 40, 9274, 248, 14000, 376] combine_rows_per_src=[9436, 7842, 16332, 8212, 16176, 15896, 16384, 16196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95292 payload_rows_remote=82000 dup_rows=35780 dup_rows_remote=30360 dedup_ratio=0.272980 router_tokens_per_expert=[101, 5299, 1736, 229, 4295, 235, 1955, 930, 2671, 220, 4264, 1524, 1340, 2228, 222, 1055, 473, 1964, 2492, 4427, 3847, 512, 5294, 2531, 58, 2638, 802, 2804, 2006, 554, 3632, 3198] payload_rows_per_dst=[11816, 11182, 12614, 13216, 13292, 10680, 10112, 12380] dup_rows_per_dst=[2914, 3648, 4744, 6840, 5420, 3264, 2550, 6400] combine_rows_per_src=[11816, 11182, 12614, 13216, 13292, 10680, 10112, 12380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119060 payload_rows_remote=104894 dup_rows=12012 dup_rows_remote=10946 dedup_ratio=0.091644 router_tokens_per_expert=[142, 8101, 1, 208, 0, 535, 54, 7027, 3, 7818, 0, 5, 232, 843, 6901, 61, 472, 0, 38, 6127, 645, 8012, 31, 2997, 8186, 0, 6094, 0, 460, 408, 0, 135] payload_rows_per_dst=[16296, 14166, 15652, 15022, 13070, 16384, 12188, 16282] dup_rows_per_dst=[608, 1066, 0, 2062, 204, 7120, 0, 952] combine_rows_per_src=[16296, 14166, 15652, 15022, 13070, 16384, 12188, 16282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=730021888 dispatch_meta_bytes=415872 combine_payload_bytes=730021888 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=730437760 ingress_bytes=1434664960 link_bytes=1434664960 token_bytes=1460459648 weight_movement_bytes=704643072 total_nvlink_bytes=2165102720 weight_to_token_ratio=0.482480 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=17664 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.033524917e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=818888704 dispatch_meta_bytes=416808 combine_payload_bytes=818888704 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=819305512 ingress_bytes=2052014080 link_bytes=2052014080 token_bytes=1638194216 weight_movement_bytes=1233125376 total_nvlink_bytes=2871319592 weight_to_token_ratio=0.752735 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=22400 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.149042230e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122446 payload_rows_remote=106262 dup_rows=8626 dup_rows_remote=8622 dedup_ratio=0.065811 router_tokens_per_expert=[332, 7401, 2, 31, 5, 53, 0, 8073, 5, 8080, 0, 9, 330, 1633, 8176, 0, 0, 0, 1, 7838, 160, 8177, 2, 1567, 8081, 2, 5132, 134, 68, 63, 131, 50] payload_rows_per_dst=[15162, 16160, 16184, 16252, 15678, 16370, 10264, 16376] dup_rows_per_dst=[370, 102, 4, 3838, 0, 3438, 272, 602] combine_rows_per_src=[15162, 16160, 16184, 16252, 15678, 16370, 10264, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111876 payload_rows_remote=99990 dup_rows=19196 dup_rows_remote=18834 dedup_ratio=0.146454 router_tokens_per_expert=[1899, 5944, 0, 3, 21, 1837, 9, 7807, 6, 8182, 4, 1, 1395, 1126, 7854, 53, 0, 37, 120, 7332, 262, 7946, 2, 4397, 8028, 0, 1171, 2, 1, 11, 0, 86] payload_rows_per_dst=[11952, 15682, 16364, 9378, 14846, 11886, 15884, 15884] dup_rows_per_dst=[3740, 3666, 22, 4000, 132, 362, 7266, 8] combine_rows_per_src=[11952, 15682, 16364, 9378, 14846, 11886, 15884, 15884] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=739557376 dispatch_meta_bytes=458000 combine_payload_bytes=739557376 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=740015376 ingress_bytes=1091878912 link_bytes=1091878912 token_bytes=1479572752 weight_movement_bytes=352321536 total_nvlink_bytes=1831894288 weight_to_token_ratio=0.238124 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.114066135e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=859291648 dispatch_meta_bytes=463360 combine_payload_bytes=859291648 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=859755008 ingress_bytes=1563934720 link_bytes=1563934720 token_bytes=1719046656 weight_movement_bytes=704643072 total_nvlink_bytes=2423689728 weight_to_token_ratio=0.409903 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.757381614e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95032 payload_rows_remote=78700 dup_rows=36040 dup_rows_remote=19530 dedup_ratio=0.274963 router_tokens_per_expert=[0, 3361, 54, 7694, 2116, 459, 3122, 531, 1097, 2355, 5, 8172, 7609, 23, 411, 23, 5, 0, 226, 314, 0, 223, 82, 2, 6280, 0, 5, 3, 7831, 8104, 98, 5331] payload_rows_per_dst=[16332, 14916, 9212, 2110, 15470, 13708, 12572, 10712] dup_rows_per_dst=[16510, 10788, 7614, 38, 814, 126, 4, 146] combine_rows_per_src=[16332, 14916, 9212, 2110, 15470, 13708, 12572, 10712] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=870498304 dispatch_meta_bytes=459536 combine_payload_bytes=870498304 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=870957840 ingress_bytes=1222819840 link_bytes=1222819840 token_bytes=1741456144 weight_movement_bytes=352321536 total_nvlink_bytes=2093777680 weight_to_token_ratio=0.202314 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.847280674e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=819118080 dispatch_meta_bytes=475296 combine_payload_bytes=819118080 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=819593376 ingress_bytes=1523761152 link_bytes=1523761152 token_bytes=1638711456 weight_movement_bytes=704643072 total_nvlink_bytes=2343354528 weight_to_token_ratio=0.429998 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=21888 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.532426402e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=644710400 dispatch_meta_bytes=392920 combine_payload_bytes=644710400 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=645103320 ingress_bytes=1349353472 link_bytes=1349353472 token_bytes=1289813720 weight_movement_bytes=704643072 total_nvlink_bytes=1994456792 weight_to_token_ratio=0.546314 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=18176 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.555816195e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=104454 payload_rows_remote=89266 dup_rows=26618 dup_rows_remote=22530 dedup_ratio=0.203079 router_tokens_per_expert=[0, 4204, 0, 8139, 0, 0, 4010, 255, 2529, 43, 235, 8181, 5207, 1, 4566, 0, 361, 0, 34, 12, 0, 3, 1, 5, 7368, 237, 0, 1, 8126, 7783, 121, 4114] payload_rows_per_dst=[8408, 8494, 8662, 14592, 16370, 16382, 15188, 16358] dup_rows_per_dst=[0, 36, 5038, 4956, 676, 18, 4088, 11806] combine_rows_per_src=[8408, 8494, 8662, 14592, 16370, 16382, 15188, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=671744000 dispatch_meta_bytes=449440 combine_payload_bytes=671744000 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=672193440 ingress_bytes=1552547840 link_bytes=1552547840 token_bytes=1343937440 weight_movement_bytes=880803840 total_nvlink_bytes=2224741280 weight_to_token_ratio=0.655391 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=12928 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.693619839e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112394 payload_rows_remote=96416 dup_rows=18678 dup_rows_remote=12056 dedup_ratio=0.142502 router_tokens_per_expert=[104, 1375, 0, 8192, 209, 0, 6005, 3345, 2666, 981, 0, 7545, 7549, 162, 3585, 4, 1, 33, 2, 130, 0, 0, 0, 0, 7596, 27, 0, 2, 8158, 6731, 7, 1127] payload_rows_per_dst=[10208, 15134, 16316, 15978, 16308, 16372, 15246, 6832] dup_rows_per_dst=[668, 3984, 6038, 6622, 304, 0, 4, 1058] combine_rows_per_src=[10208, 15134, 16316, 15978, 16308, 16372, 15246, 6832] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96410 payload_rows_remote=81086 dup_rows=34662 dup_rows_remote=26696 dedup_ratio=0.264450 router_tokens_per_expert=[62, 6183, 17, 8168, 1776, 16, 0, 5781, 1130, 658, 985, 8085, 8132, 3, 1374, 0, 12, 2, 1, 57, 2, 24, 0, 9, 6516, 33, 0, 45, 8183, 7905, 50, 327] payload_rows_per_dst=[16380, 15324, 9956, 8126, 16376, 16358, 13136, 754] dup_rows_per_dst=[20146, 7966, 3860, 2488, 110, 40, 52, 0] combine_rows_per_src=[16380, 15324, 9956, 8126, 16376, 16358, 13136, 754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=102096 payload_rows_remote=85722 dup_rows=28976 dup_rows_remote=12960 dedup_ratio=0.221069 router_tokens_per_expert=[40, 3869, 7, 8127, 1, 2, 589, 3500, 193, 726, 194, 8176, 6622, 25, 7339, 7, 0, 0, 0, 11, 145, 5, 0, 20, 7622, 10, 3, 2, 8145, 8044, 264, 1848] payload_rows_per_dst=[7758, 7714, 8370, 15746, 14622, 16236, 15276, 16374] dup_rows_per_dst=[74, 470, 1920, 5648, 16, 300, 4532, 16016] combine_rows_per_src=[7758, 7714, 8370, 15746, 14622, 16236, 15276, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=731267072 dispatch_meta_bytes=447184 combine_payload_bytes=731267072 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=731714256 ingress_bytes=1083588608 link_bytes=1083588608 token_bytes=1462981328 weight_movement_bytes=352321536 total_nvlink_bytes=1815302864 weight_to_token_ratio=0.240824 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.067643893e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115018 payload_rows_remote=101358 dup_rows=16054 dup_rows_remote=12074 dedup_ratio=0.122482 router_tokens_per_expert=[89, 15, 0, 8171, 22, 14, 5139, 4414, 233, 3687, 46, 8184, 6996, 0, 3634, 1, 17, 0, 193, 0, 18, 0, 0, 24, 8052, 1, 0, 0, 8158, 8038, 116, 274] payload_rows_per_dst=[16320, 15808, 13660, 12652, 16374, 16376, 16106, 7722] dup_rows_per_dst=[1018, 6352, 3980, 4112, 364, 74, 0, 154] combine_rows_per_src=[16320, 15808, 13660, 12652, 16374, 16376, 16106, 7722] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=789839872 dispatch_meta_bytes=433888 combine_payload_bytes=789839872 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=790273760 ingress_bytes=1670643712 link_bytes=1670643712 token_bytes=1580113632 weight_movement_bytes=880803840 total_nvlink_bytes=2460917472 weight_to_token_ratio=0.557431 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.354907426e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110974 payload_rows_remote=94772 dup_rows=20098 dup_rows_remote=19918 dedup_ratio=0.153336 router_tokens_per_expert=[421, 3911, 42, 8134, 2, 14, 3200, 3969, 90, 390, 30, 8108, 5474, 0, 4700, 1, 108, 0, 2, 32, 0, 0, 0, 107, 8160, 0, 6, 0, 8049, 6856, 1003, 2727] payload_rows_per_dst=[8664, 11308, 13480, 13242, 16274, 16202, 16342, 15462] dup_rows_per_dst=[84, 3062, 236, 7108, 132, 180, 3586, 5710] combine_rows_per_src=[8664, 11308, 13480, 13242, 16274, 16202, 16342, 15462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=664256512 dispatch_meta_bytes=431128 combine_payload_bytes=664256512 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=664687640 ingress_bytes=1016578048 link_bytes=1016578048 token_bytes=1328944152 weight_movement_bytes=352321536 total_nvlink_bytes=1681265688 weight_to_token_ratio=0.265114 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=15488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.692412728e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=702234624 dispatch_meta_bytes=394728 combine_payload_bytes=702234624 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=702629352 ingress_bytes=1759199232 link_bytes=1759199232 token_bytes=1404863976 weight_movement_bytes=1056964608 total_nvlink_bytes=2461828584 weight_to_token_ratio=0.752361 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=21632 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.850781373e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98342 payload_rows_remote=82702 dup_rows=32730 dup_rows_remote=25372 dedup_ratio=0.249710 router_tokens_per_expert=[28, 2, 0, 3657, 1, 1952, 212, 190, 127, 1, 2, 1930, 8168, 7641, 7812, 12, 19, 3451, 647, 659, 2293, 399, 6988, 2, 3839, 6976, 478, 31, 4, 7303, 712, 0] payload_rows_per_dst=[15640, 16364, 12754, 15252, 9778, 15936, 8200, 4418] dup_rows_per_dst=[7358, 2416, 4070, 6380, 7846, 4164, 496, 0] combine_rows_per_src=[15640, 16364, 12754, 15252, 9778, 15936, 8200, 4418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=830324736 dispatch_meta_bytes=453728 combine_payload_bytes=830324736 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=830778464 ingress_bytes=1711128576 link_bytes=1711128576 token_bytes=1661103200 weight_movement_bytes=880803840 total_nvlink_bytes=2541907040 weight_to_token_ratio=0.530252 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=22144 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.581605765e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=776372224 dispatch_meta_bytes=458760 combine_payload_bytes=776372224 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=776830984 ingress_bytes=1128693760 link_bytes=1128693760 token_bytes=1553203208 weight_movement_bytes=352321536 total_nvlink_bytes=1905524744 weight_to_token_ratio=0.226835 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.320213916e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=677494784 dispatch_meta_bytes=432296 combine_payload_bytes=677494784 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=677927080 ingress_bytes=1029816320 link_bytes=1029816320 token_bytes=1355421864 weight_movement_bytes=352321536 total_nvlink_bytes=1707743400 weight_to_token_ratio=0.259935 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=28288 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.766541526e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=102570 payload_rows_remote=86998 dup_rows=28502 dup_rows_remote=11316 dedup_ratio=0.217453 router_tokens_per_expert=[0, 0, 1, 398, 0, 0, 142, 386, 11, 2360, 0, 3201, 7674, 8156, 8190, 23, 0, 3560, 57, 6, 3434, 83, 1920, 140, 6079, 8144, 11, 0, 14, 8179, 3365, 2] payload_rows_per_dst=[8872, 11994, 10204, 14918, 15274, 9356, 15572, 16380] dup_rows_per_dst=[732, 964, 940, 50, 92, 1798, 17186, 6740] combine_rows_per_src=[8872, 11994, 10204, 14918, 15274, 9356, 15572, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111486 payload_rows_remote=95292 dup_rows=19586 dup_rows_remote=16768 dedup_ratio=0.149429 router_tokens_per_expert=[9, 588, 8113, 54, 0, 4, 3592, 88, 137, 40, 27, 370, 1, 25, 2, 1, 1447, 4, 1, 7, 7892, 0, 6239, 59, 7770, 4478, 8022, 163, 7777, 1, 1869, 6756] payload_rows_per_dst=[16244, 14110, 15422, 16340, 16194, 15836, 8966, 8374] dup_rows_per_dst=[1284, 3422, 454, 34, 2818, 9762, 316, 1496] combine_rows_per_src=[16244, 14110, 15422, 16340, 16194, 15836, 8966, 8374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107416 payload_rows_remote=91974 dup_rows=23656 dup_rows_remote=23656 dedup_ratio=0.180481 router_tokens_per_expert=[0, 0, 6, 215, 0, 10, 19, 2806, 41, 1755, 0, 5413, 8188, 8063, 8192, 3, 0, 4015, 12, 0, 8135, 0, 795, 0, 1833, 7857, 0, 0, 0, 8178, 0, 0] payload_rows_per_dst=[16316, 16384, 14386, 15442, 15936, 8930, 3666, 16356] dup_rows_per_dst=[434, 5670, 9050, 0, 7650, 852, 0, 0] combine_rows_per_src=[16316, 16384, 14386, 15442, 15936, 8930, 3666, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112514 payload_rows_remote=96144 dup_rows=18558 dup_rows_remote=17552 dedup_ratio=0.141586 router_tokens_per_expert=[0, 43, 83, 144, 5, 139, 671, 3, 12, 2882, 0, 19, 8070, 8118, 8107, 4, 84, 5706, 49, 0, 1717, 0, 7297, 7, 6323, 7817, 38, 0, 0, 8185, 10, 3] payload_rows_per_dst=[16300, 16370, 11252, 13842, 14422, 14868, 15054, 10406] dup_rows_per_dst=[308, 1006, 5814, 0, 7646, 156, 3624, 4] combine_rows_per_src=[16300, 16370, 11252, 13842, 14422, 14868, 15054, 10406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=712687616 dispatch_meta_bytes=393256 combine_payload_bytes=712687616 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=713080872 ingress_bytes=1417330688 link_bytes=1417330688 token_bytes=1425768488 weight_movement_bytes=704643072 total_nvlink_bytes=2130411560 weight_to_token_ratio=0.494220 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=23040 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.936460230e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=91268 payload_rows_remote=74886 dup_rows=39804 dup_rows_remote=34366 dedup_ratio=0.303680 router_tokens_per_expert=[0, 0, 2, 288, 43, 28, 0, 12, 2, 66, 3, 4704, 7571, 8111, 8169, 0, 0, 1245, 2, 2, 3637, 86, 6809, 547, 5998, 8189, 2, 0, 0, 8172, 1848, 0] payload_rows_per_dst=[576, 7214, 9418, 16382, 8606, 16306, 16384, 16382] dup_rows_per_dst=[4, 60, 132, 8036, 1924, 5852, 18358, 5438] combine_rows_per_src=[576, 7214, 9418, 16382, 8606, 16306, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 1, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114038 payload_rows_remote=97802 dup_rows=17034 dup_rows_remote=9060 dedup_ratio=0.129959 router_tokens_per_expert=[1, 26, 39, 2665, 11, 176, 0, 58, 9, 33, 0, 5549, 8132, 8103, 8192, 76, 0, 800, 0, 7, 5894, 0, 4347, 226, 4917, 8166, 0, 0, 0, 8108, 1, 0] payload_rows_per_dst=[16288, 16334, 16236, 8328, 16374, 14426, 9834, 16218] dup_rows_per_dst=[5404, 488, 7974, 0, 1600, 1568, 0, 0] combine_rows_per_src=[16288, 16334, 16236, 8328, 16374, 14426, 9834, 16218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105490 payload_rows_remote=89970 dup_rows=25582 dup_rows_remote=17518 dedup_ratio=0.195175 router_tokens_per_expert=[3807, 16, 204, 341, 7, 1, 323, 5, 8159, 0, 55, 1184, 0, 148, 319, 5, 6683, 5299, 5388, 1, 0, 28, 41, 7980, 7017, 6279, 9, 0, 7901, 1, 4327, 8] payload_rows_per_dst=[15520, 15804, 16384, 11390, 16220, 16108, 5392, 8672] dup_rows_per_dst=[8064, 658, 3650, 342, 7034, 5834, 0, 0] combine_rows_per_src=[15520, 15804, 16384, 11390, 16220, 16108, 5392, 8672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=753451008 dispatch_meta_bytes=462520 combine_payload_bytes=753451008 prefetch_bytes=176160768 reduce_bytes=1233125376 egress_bytes=753913528 ingress_bytes=2162737152 link_bytes=2162737152 token_bytes=1507364536 weight_movement_bytes=1409286144 total_nvlink_bytes=2916650680 weight_to_token_ratio=0.934934 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=24320 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.211042528e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=787611648 dispatch_meta_bytes=454784 combine_payload_bytes=787611648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=788066432 ingress_bytes=1139933184 link_bytes=1139933184 token_bytes=1575678080 weight_movement_bytes=352321536 total_nvlink_bytes=1927999616 weight_to_token_ratio=0.223600 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.383149999e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=780632064 dispatch_meta_bytes=448240 combine_payload_bytes=780632064 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=781080304 ingress_bytes=1132953600 link_bytes=1132953600 token_bytes=1561712368 weight_movement_bytes=352321536 total_nvlink_bytes=1914033904 weight_to_token_ratio=0.225600 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.344067242e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115298 payload_rows_remote=105462 dup_rows=15774 dup_rows_remote=11094 dedup_ratio=0.120346 router_tokens_per_expert=[0, 0, 0, 1611, 98, 83, 0, 657, 8, 2831, 0, 5518, 8051, 8192, 8136, 0, 11, 5, 749, 0, 3444, 0, 3814, 0, 3749, 7914, 49, 1, 0, 8095, 2498, 22] payload_rows_per_dst=[16384, 16378, 15078, 16102, 9692, 9836, 15596, 16232] dup_rows_per_dst=[3220, 1042, 1636, 0, 30, 4680, 168, 4998] combine_rows_per_src=[16384, 16378, 15078, 16102, 9692, 9836, 15596, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=613466112 dispatch_meta_bytes=437008 combine_payload_bytes=613466112 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=613903120 ingress_bytes=1494269952 link_bytes=1494269952 token_bytes=1227369232 weight_movement_bytes=880803840 total_nvlink_bytes=2108173072 weight_to_token_ratio=0.717636 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=30080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.367287992e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=801193984 dispatch_meta_bytes=427448 combine_payload_bytes=801193984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=801621432 ingress_bytes=1153515520 link_bytes=1153515520 token_bytes=1602815416 weight_movement_bytes=352321536 total_nvlink_bytes=1955136952 weight_to_token_ratio=0.219814 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=18432 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.459205411e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=737034240 dispatch_meta_bytes=429952 combine_payload_bytes=737034240 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=737464192 ingress_bytes=1089355776 link_bytes=1089355776 token_bytes=1474498432 weight_movement_bytes=352321536 total_nvlink_bytes=1826819968 weight_to_token_ratio=0.238943 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=19584 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.099937626e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118156 payload_rows_remote=101776 dup_rows=12916 dup_rows_remote=4902 dedup_ratio=0.098541 router_tokens_per_expert=[3514, 3, 5077, 277, 0, 0, 391, 2, 8189, 4, 22, 0, 21, 4, 3, 303, 3, 6018, 8112, 0, 0, 43, 0, 5317, 8188, 3904, 105, 0, 8152, 3, 7835, 46] payload_rows_per_dst=[14104, 16374, 16384, 16384, 12042, 10720, 16380, 15768] dup_rows_per_dst=[3638, 712, 46, 506, 0, 0, 8014, 0] combine_rows_per_src=[14104, 16374, 16384, 16384, 12042, 10720, 16380, 15768] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=863944704 dispatch_meta_bytes=466224 combine_payload_bytes=863944704 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=864410928 ingress_bytes=1392427008 link_bytes=1392427008 token_bytes=1728355632 weight_movement_bytes=528482304 total_nvlink_bytes=2256837936 weight_to_token_ratio=0.305772 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=20864 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.797010017e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119306 payload_rows_remote=102930 dup_rows=11766 dup_rows_remote=11042 dedup_ratio=0.089767 router_tokens_per_expert=[4096, 0, 520, 12, 588, 0, 1011, 0, 7966, 6, 0, 216, 0, 150, 39, 188, 958, 6884, 7243, 0, 0, 95, 16, 4104, 7923, 7202, 24, 4, 8192, 0, 8099, 0] payload_rows_per_dst=[8550, 16046, 16188, 16376, 15104, 16384, 14460, 16198] dup_rows_per_dst=[706, 1676, 188, 724, 580, 7892, 0, 0] combine_rows_per_src=[8550, 16046, 16188, 16376, 15104, 16384, 14460, 16198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112394 payload_rows_remote=97066 dup_rows=18678 dup_rows_remote=18440 dedup_ratio=0.142502 router_tokens_per_expert=[4362, 123, 3054, 1, 1, 3, 8, 527, 8116, 0, 5, 1269, 1, 0, 0, 1, 3003, 6726, 7249, 0, 32, 46, 0, 4090, 7333, 3241, 7, 566, 8178, 4, 7570, 20] payload_rows_per_dst=[13118, 15328, 16384, 16368, 14802, 14824, 6382, 15188] dup_rows_per_dst=[4074, 238, 3296, 2496, 7832, 742, 0, 0] combine_rows_per_src=[13118, 15328, 16384, 16368, 14802, 14824, 6382, 15188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=833748992 dispatch_meta_bytes=426712 combine_payload_bytes=833748992 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=834175704 ingress_bytes=1890713600 link_bytes=1890713600 token_bytes=1667924696 weight_movement_bytes=1056964608 total_nvlink_bytes=2724889304 weight_to_token_ratio=0.633700 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.058720694e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97908 payload_rows_remote=82268 dup_rows=33164 dup_rows_remote=16538 dedup_ratio=0.253021 router_tokens_per_expert=[6330, 11, 2483, 1540, 0, 10, 934, 0, 8105, 46, 4, 1, 0, 489, 583, 107, 180, 8114, 8156, 26, 2, 138, 0, 4459, 8117, 286, 8, 0, 8174, 22, 7191, 20] payload_rows_per_dst=[15860, 6966, 9772, 13028, 11356, 9032, 16254, 15640] dup_rows_per_dst=[4868, 1666, 12, 1866, 2472, 166, 5488, 16626] combine_rows_per_src=[15860, 6966, 9772, 13028, 11356, 9032, 16254, 15640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115634 payload_rows_remote=99928 dup_rows=15438 dup_rows_remote=15288 dedup_ratio=0.117783 router_tokens_per_expert=[655, 7702, 0, 221, 2, 2181, 9, 7544, 0, 7809, 0, 235, 2555, 219, 8185, 0, 38, 50, 3, 7837, 592, 8188, 8, 1046, 8028, 8, 2181, 13, 105, 0, 0, 122] payload_rows_per_dst=[16264, 15360, 15726, 16070, 15706, 11086, 9042, 16380] dup_rows_per_dst=[892, 4112, 362, 5532, 150, 390, 3546, 454] combine_rows_per_src=[16264, 15360, 15726, 16070, 15706, 11086, 9042, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115908 payload_rows_remote=103414 dup_rows=15164 dup_rows_remote=15102 dedup_ratio=0.115692 router_tokens_per_expert=[0, 14, 5424, 162, 5, 8178, 68, 0, 517, 484, 216, 5788, 945, 8039, 4070, 22, 21, 678, 3, 7601, 0, 7442, 3, 0, 7657, 0, 0, 1, 5, 97, 7988, 108] payload_rows_per_dst=[12494, 16384, 16380, 10032, 15248, 14884, 14464, 16022] dup_rows_per_dst=[62, 912, 13708, 42, 60, 6, 0, 374] combine_rows_per_src=[12494, 16384, 16380, 10032, 15248, 14884, 14464, 16022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116664 payload_rows_remote=100292 dup_rows=14408 dup_rows_remote=14408 dedup_ratio=0.109924 router_tokens_per_expert=[5541, 0, 1207, 48, 4, 0, 1565, 25, 8185, 0, 0, 1, 14, 0, 429, 0, 799, 6092, 8187, 8, 1, 0, 43, 4219, 8082, 4811, 4, 0, 8136, 0, 8135, 0] payload_rows_per_dst=[12896, 16382, 16372, 16384, 12358, 16376, 9626, 16270] dup_rows_per_dst=[696, 3074, 0, 880, 1440, 8314, 4, 0] combine_rows_per_src=[12896, 16382, 16372, 16384, 12358, 16376, 9626, 16270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=795164672 dispatch_meta_bytes=462024 combine_payload_bytes=795164672 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=795626696 ingress_bytes=1147486208 link_bytes=1147486208 token_bytes=1590791368 weight_movement_bytes=352321536 total_nvlink_bytes=1943112904 weight_to_token_ratio=0.221476 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=29696 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.425443780e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=843202560 dispatch_meta_bytes=455888 combine_payload_bytes=843202560 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=843658448 ingress_bytes=1371684864 link_bytes=1371684864 token_bytes=1686861008 weight_movement_bytes=528482304 total_nvlink_bytes=2215343312 weight_to_token_ratio=0.313293 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.020833 gemm_active_groups=13 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.680862669e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113718 payload_rows_remote=101130 dup_rows=17354 dup_rows_remote=17268 dedup_ratio=0.132401 router_tokens_per_expert=[2670, 236, 4258, 190, 10, 227, 1390, 174, 7959, 1, 0, 0, 12, 60, 515, 166, 0, 4079, 8090, 4, 0, 359, 12, 5966, 7862, 5217, 22, 0, 8074, 0, 7983, 0] payload_rows_per_dst=[12046, 16384, 15920, 16376, 8166, 12588, 16272, 15966] dup_rows_per_dst=[2662, 3334, 0, 1342, 0, 86, 9930, 0] combine_rows_per_src=[12046, 16384, 15920, 16376, 8166, 12588, 16272, 15966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=673939456 dispatch_meta_bytes=395224 combine_payload_bytes=673939456 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=674334680 ingress_bytes=1378582528 link_bytes=1378582528 token_bytes=1348274136 weight_movement_bytes=704643072 total_nvlink_bytes=2052917208 weight_to_token_ratio=0.522626 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.719486708e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=847167488 dispatch_meta_bytes=474064 combine_payload_bytes=847167488 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=847641552 ingress_bytes=1199489024 link_bytes=1199489024 token_bytes=1694809040 weight_movement_bytes=352321536 total_nvlink_bytes=2047130576 weight_to_token_ratio=0.207883 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=18048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.716637843e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=818610176 dispatch_meta_bytes=460864 combine_payload_bytes=818610176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=819071040 ingress_bytes=1170931712 link_bytes=1170931712 token_bytes=1637681216 weight_movement_bytes=352321536 total_nvlink_bytes=1990002752 weight_to_token_ratio=0.215134 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.556728816e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107316 payload_rows_remote=91024 dup_rows=23756 dup_rows_remote=23734 dedup_ratio=0.181244 router_tokens_per_expert=[1, 0, 4206, 1114, 58, 8178, 747, 1, 0, 0, 0, 2274, 1, 8125, 8023, 8, 13, 4797, 53, 8179, 4, 3292, 0, 57, 8146, 4, 7, 0, 0, 36, 8174, 38] payload_rows_per_dst=[8516, 16384, 7598, 16196, 9598, 16384, 16292, 16348] dup_rows_per_dst=[2126, 1584, 4176, 8892, 128, 6680, 22, 148] combine_rows_per_src=[8516, 16384, 7598, 16196, 9598, 16384, 16292, 16348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=821592064 dispatch_meta_bytes=458800 combine_payload_bytes=821592064 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=822050864 ingress_bytes=1526235136 link_bytes=1526235136 token_bytes=1643642928 weight_movement_bytes=704643072 total_nvlink_bytes=2348286000 weight_to_token_ratio=0.428708 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.546279680e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=828456960 dispatch_meta_bytes=473592 combine_payload_bytes=828456960 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=828930552 ingress_bytes=1180778496 link_bytes=1180778496 token_bytes=1657387512 weight_movement_bytes=352321536 total_nvlink_bytes=2009709048 weight_to_token_ratio=0.212576 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=23808 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.611866696e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114166 payload_rows_remote=97854 dup_rows=16906 dup_rows_remote=14742 dedup_ratio=0.128983 router_tokens_per_expert=[124, 0, 5762, 1350, 0, 8050, 1188, 0, 0, 0, 61, 106, 0, 7791, 7820, 0, 30, 7855, 0, 8191, 429, 402, 2, 161, 8032, 3, 31, 73, 6, 2, 8054, 13] payload_rows_per_dst=[16336, 16312, 15690, 15640, 16382, 1412, 16246, 16148] dup_rows_per_dst=[13846, 2164, 226, 0, 60, 576, 32, 2] combine_rows_per_src=[16336, 16312, 15690, 15640, 16382, 1412, 16246, 16148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=745668608 dispatch_meta_bytes=459032 combine_payload_bytes=745668608 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=746127640 ingress_bytes=1274150912 link_bytes=1274150912 token_bytes=1491796248 weight_movement_bytes=528482304 total_nvlink_bytes=2020278552 weight_to_token_ratio=0.354259 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.134713250e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118368 payload_rows_remote=102282 dup_rows=12704 dup_rows_remote=12686 dedup_ratio=0.096924 router_tokens_per_expert=[0, 0, 6342, 637, 76, 8005, 238, 0, 2, 204, 0, 320, 9, 8164, 8043, 0, 2, 7514, 0, 8045, 0, 2103, 0, 0, 8184, 7, 0, 5, 99, 0, 7537, 0] payload_rows_per_dst=[15638, 16038, 16376, 16086, 16098, 6680, 16378, 15074] dup_rows_per_dst=[5346, 600, 1004, 18, 5348, 176, 14, 198] combine_rows_per_src=[15638, 16038, 16376, 16086, 16098, 6680, 16378, 15074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105640 payload_rows_remote=89700 dup_rows=25432 dup_rows_remote=24038 dedup_ratio=0.194031 router_tokens_per_expert=[279, 11, 3153, 1870, 15, 8124, 0, 0, 67, 0, 0, 953, 0, 7267, 7115, 2, 62, 7850, 72, 8127, 50, 4247, 1, 0, 8084, 28, 2, 115, 0, 19, 7929, 94] payload_rows_per_dst=[10422, 12266, 2040, 16284, 15998, 16354, 16336, 15940] dup_rows_per_dst=[204, 30, 0, 10414, 3476, 8496, 1418, 1394] combine_rows_per_src=[10422, 12266, 2040, 16284, 15998, 16354, 16336, 15940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=801619968 dispatch_meta_bytes=450384 combine_payload_bytes=801619968 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=802070352 ingress_bytes=1506263040 link_bytes=1506263040 token_bytes=1603690320 weight_movement_bytes=704643072 total_nvlink_bytes=2308333392 weight_to_token_ratio=0.439388 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.434444279e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110360 payload_rows_remote=94290 dup_rows=20712 dup_rows_remote=11854 dedup_ratio=0.158020 router_tokens_per_expert=[6875, 28, 8, 0, 7731, 8016, 587, 1, 2, 1, 407, 1, 0, 27, 4468, 1, 131, 317, 1, 0, 48, 131, 8161, 8186, 0, 2528, 0, 0, 306, 8191, 3464, 5919] payload_rows_per_dst=[16070, 13804, 14826, 14850, 12472, 15926, 12632, 9780] dup_rows_per_dst=[8858, 3580, 2, 1710, 264, 260, 58, 5980] combine_rows_per_src=[16070, 13804, 14826, 14850, 12472, 15926, 12632, 9780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=837894144 dispatch_meta_bytes=459872 combine_payload_bytes=837894144 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=838354016 ingress_bytes=1894858752 link_bytes=1894858752 token_bytes=1676248160 weight_movement_bytes=1056964608 total_nvlink_bytes=2733212768 weight_to_token_ratio=0.630554 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=27008 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.061041806e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117474 payload_rows_remote=101128 dup_rows=13598 dup_rows_remote=6956 dedup_ratio=0.103745 router_tokens_per_expert=[375, 10, 301, 4277, 0, 7972, 225, 0, 3, 0, 0, 3548, 49, 7943, 7046, 14, 1, 4887, 0, 8172, 865, 3411, 0, 0, 8192, 10, 0, 0, 13, 97, 8125, 0] payload_rows_per_dst=[13546, 16342, 16346, 14158, 16346, 7978, 16384, 16374] dup_rows_per_dst=[3110, 52, 6642, 60, 3044, 574, 20, 96] combine_rows_per_src=[13546, 16342, 16346, 14158, 16346, 7978, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117350 payload_rows_remote=100970 dup_rows=13722 dup_rows_remote=4168 dedup_ratio=0.104691 router_tokens_per_expert=[1711, 0, 4868, 2011, 2, 7898, 0, 11, 0, 91, 12, 1217, 11, 5942, 4094, 0, 245, 8147, 0, 8175, 6, 4781, 0, 5, 7980, 0, 50, 2, 44, 90, 8143, 0] payload_rows_per_dst=[15092, 15802, 13058, 8208, 16374, 16380, 16064, 16372] dup_rows_per_dst=[2088, 20, 260, 1208, 410, 9554, 0, 182] combine_rows_per_src=[15092, 15802, 13058, 8208, 16374, 16380, 16064, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=734822400 dispatch_meta_bytes=454952 combine_payload_bytes=734822400 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=735277352 ingress_bytes=1087143936 link_bytes=1087143936 token_bytes=1470099752 weight_movement_bytes=352321536 total_nvlink_bytes=1822421288 weight_to_token_ratio=0.239658 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.087552245e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111016 payload_rows_remote=94848 dup_rows=20056 dup_rows_remote=19362 dedup_ratio=0.153015 router_tokens_per_expert=[8, 1996, 0, 8076, 1298, 0, 1841, 1719, 1166, 2025, 0, 8062, 2086, 0, 6444, 297, 1, 4, 317, 29, 1248, 38, 2, 5, 7468, 0, 0, 2, 8173, 7667, 298, 5266] payload_rows_per_dst=[8146, 8900, 16380, 14258, 16168, 16346, 14940, 15878] dup_rows_per_dst=[356, 816, 6350, 3396, 694, 2354, 0, 6090] combine_rows_per_src=[8146, 8900, 16380, 14258, 16168, 16346, 14940, 15878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=772423680 dispatch_meta_bytes=424576 combine_payload_bytes=772423680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=772848256 ingress_bytes=1124745216 link_bytes=1124745216 token_bytes=1545271936 weight_movement_bytes=352321536 total_nvlink_bytes=1897593472 weight_to_token_ratio=0.228000 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.298103718e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117558 payload_rows_remote=101180 dup_rows=13514 dup_rows_remote=13182 dedup_ratio=0.103104 router_tokens_per_expert=[8087, 0, 0, 5, 8166, 8182, 280, 8, 0, 4, 2, 0, 9, 0, 743, 1106, 5, 0, 0, 14, 0, 65, 8190, 8153, 1, 198, 3, 0, 3, 8018, 6404, 7890] payload_rows_per_dst=[16182, 16380, 16344, 3710, 15808, 16380, 16378, 16376] dup_rows_per_dst=[2, 556, 4, 6, 10, 130, 332, 12474] combine_rows_per_src=[16182, 16380, 16344, 3710, 15808, 16380, 16378, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=828440576 dispatch_meta_bytes=432336 combine_payload_bytes=828440576 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=828872912 ingress_bytes=1180762112 link_bytes=1180762112 token_bytes=1657313488 weight_movement_bytes=352321536 total_nvlink_bytes=2009635024 weight_to_token_ratio=0.212586 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=25728 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.611774953e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=827146240 dispatch_meta_bytes=420552 combine_payload_bytes=827146240 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=827566792 ingress_bytes=1179467776 link_bytes=1179467776 token_bytes=1654713032 weight_movement_bytes=352321536 total_nvlink_bytes=2007034568 weight_to_token_ratio=0.212920 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=17920 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.604527212e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118722 payload_rows_remote=102710 dup_rows=12350 dup_rows_remote=12222 dedup_ratio=0.094223 router_tokens_per_expert=[8163, 0, 1, 0, 8094, 7702, 136, 12, 34, 0, 856, 4, 255, 0, 5717, 3044, 23, 29, 0, 26, 0, 676, 8183, 8126, 0, 119, 11, 233, 0, 5428, 1680, 6984] payload_rows_per_dst=[16326, 16012, 16316, 12020, 14060, 16374, 16384, 11230] dup_rows_per_dst=[2, 128, 1554, 6012, 64, 1344, 260, 2986] combine_rows_per_src=[16326, 16012, 16316, 12020, 14060, 16374, 16384, 11230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125444 payload_rows_remote=111184 dup_rows=5628 dup_rows_remote=4138 dedup_ratio=0.042938 router_tokens_per_expert=[7599, 0, 0, 0, 7812, 8171, 755, 2, 0, 0, 0, 0, 3181, 0, 4685, 9, 13, 0, 0, 0, 53, 0, 8192, 8188, 0, 903, 0, 4, 0, 6942, 847, 8180] payload_rows_per_dst=[15198, 15624, 16376, 14260, 16382, 16384, 16352, 14868] dup_rows_per_dst=[0, 1512, 0, 1490, 4, 106, 1806, 710] combine_rows_per_src=[15198, 15624, 16376, 14260, 16382, 16384, 16352, 14868] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=776994816 dispatch_meta_bytes=456840 combine_payload_bytes=776994816 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=777451656 ingress_bytes=1129316352 link_bytes=1129316352 token_bytes=1554446472 weight_movement_bytes=352321536 total_nvlink_bytes=1906768008 weight_to_token_ratio=0.226654 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.323700171e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=828866560 dispatch_meta_bytes=457448 combine_payload_bytes=828866560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=829324008 ingress_bytes=1181188096 link_bytes=1181188096 token_bytes=1658190568 weight_movement_bytes=352321536 total_nvlink_bytes=2010512104 weight_to_token_ratio=0.212473 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.614160285e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116142 payload_rows_remote=101376 dup_rows=14930 dup_rows_remote=5474 dedup_ratio=0.113907 router_tokens_per_expert=[7943, 37, 0, 95, 8145, 8139, 538, 306, 12, 0, 0, 30, 3596, 3, 3815, 98, 29, 4, 18, 8, 431, 0, 7789, 8163, 49, 1, 5, 5, 0, 3863, 4522, 7892] payload_rows_per_dst=[16078, 13908, 16256, 14284, 10054, 14420, 16376, 14766] dup_rows_per_dst=[72, 1598, 26, 3214, 14, 496, 54, 9456] combine_rows_per_src=[16078, 13908, 16256, 14284, 10054, 14420, 16376, 14766] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=841400320 dispatch_meta_bytes=459728 combine_payload_bytes=841400320 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=841860048 ingress_bytes=1898364928 link_bytes=1898364928 token_bytes=1683260368 weight_movement_bytes=1056964608 total_nvlink_bytes=2740224976 weight_to_token_ratio=0.627927 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.063005118e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107102 payload_rows_remote=90720 dup_rows=23970 dup_rows_remote=15334 dedup_ratio=0.182877 router_tokens_per_expert=[3795, 28, 368, 215, 23, 8, 55, 2984, 124, 552, 5, 11, 0, 1, 7, 4101, 6341, 8084, 0, 0, 7996, 6083, 7995, 79, 4, 735, 0, 18, 1, 4732, 4062, 7129] payload_rows_per_dst=[16382, 16304, 14512, 9476, 13930, 15104, 13268, 8126] dup_rows_per_dst=[8636, 5992, 1150, 2608, 4350, 1234, 0, 0] combine_rows_per_src=[16382, 16304, 14512, 9476, 13930, 15104, 13268, 8126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=910819328 dispatch_meta_bytes=461288 combine_payload_bytes=910819328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=911280616 ingress_bytes=1263140864 link_bytes=1263140864 token_bytes=1822099944 weight_movement_bytes=352321536 total_nvlink_bytes=2174421480 weight_to_token_ratio=0.193360 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=24704 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.073061577e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121454 payload_rows_remote=105082 dup_rows=9618 dup_rows_remote=9618 dedup_ratio=0.073380 router_tokens_per_expert=[8176, 33, 0, 0, 8176, 8161, 13, 8, 0, 1, 9, 0, 2195, 0, 6140, 0, 2690, 0, 0, 88, 129, 0, 7966, 8127, 18, 619, 0, 0, 0, 728, 4099, 8160] payload_rows_per_dst=[16382, 16326, 16372, 14038, 16374, 16084, 16290, 9588] dup_rows_per_dst=[36, 42, 0, 2632, 5502, 106, 1234, 66] combine_rows_per_src=[16382, 16326, 16372, 14038, 16374, 16084, 16290, 9588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=830472192 dispatch_meta_bytes=427400 combine_payload_bytes=830472192 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=830899592 ingress_bytes=1887436800 link_bytes=1887436800 token_bytes=1661371784 weight_movement_bytes=1056964608 total_nvlink_bytes=2718336392 weight_to_token_ratio=0.636200 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=18688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.056885823e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121318 payload_rows_remote=106834 dup_rows=9754 dup_rows_remote=9544 dedup_ratio=0.074417 router_tokens_per_expert=[5526, 0, 0, 0, 7998, 8063, 4, 0, 38, 0, 0, 1, 0, 47, 7846, 1979, 2323, 5, 0, 15, 53, 85, 7209, 8142, 0, 0, 0, 81, 1, 6193, 1750, 8177] payload_rows_per_dst=[11052, 16006, 16372, 16358, 16382, 14484, 16198, 14466] dup_rows_per_dst=[0, 0, 76, 3386, 4658, 210, 2, 1422] combine_rows_per_src=[11052, 16006, 16372, 16358, 16382, 14484, 16198, 14466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=743178240 dispatch_meta_bytes=424216 combine_payload_bytes=743178240 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=743602456 ingress_bytes=1095499776 link_bytes=1095499776 token_bytes=1486780696 weight_movement_bytes=352321536 total_nvlink_bytes=1839102232 weight_to_token_ratio=0.236969 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=25088 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.134341461e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=860831744 dispatch_meta_bytes=458800 combine_payload_bytes=860831744 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=861290544 ingress_bytes=1213153280 link_bytes=1213153280 token_bytes=1722122288 weight_movement_bytes=352321536 total_nvlink_bytes=2074443824 weight_to_token_ratio=0.204586 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.793151973e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113054 payload_rows_remote=96672 dup_rows=18018 dup_rows_remote=17936 dedup_ratio=0.137466 router_tokens_per_expert=[4609, 33, 6, 0, 5, 0, 0, 15, 3, 97, 2010, 1, 148, 6, 1, 7793, 8188, 8185, 0, 2, 3455, 6141, 8185, 14, 0, 41, 5, 0, 1, 8146, 262, 8184] payload_rows_per_dst=[12044, 8202, 16384, 15604, 16376, 11768, 16382, 16294] dup_rows_per_dst=[420, 28, 4206, 292, 7838, 4628, 82, 524] combine_rows_per_src=[12044, 8202, 16384, 15604, 16376, 11768, 16382, 16294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=875184128 dispatch_meta_bytes=465512 combine_payload_bytes=875184128 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=875649640 ingress_bytes=1932148736 link_bytes=1932148736 token_bytes=1750833768 weight_movement_bytes=1056964608 total_nvlink_bytes=2807798376 weight_to_token_ratio=0.603692 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.024621 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.081922640e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115404 payload_rows_remote=99088 dup_rows=15668 dup_rows_remote=15540 dedup_ratio=0.119537 router_tokens_per_expert=[38, 4, 0, 49, 31, 4, 2, 34, 90, 23, 0, 1, 0, 21, 680, 8056, 6933, 8153, 1, 12, 6090, 8093, 8174, 0, 2, 316, 1, 37, 73, 8162, 2314, 8142] payload_rows_per_dst=[16286, 16316, 16378, 16378, 16350, 12180, 16374, 5142] dup_rows_per_dst=[82, 128, 228, 8926, 5698, 0, 596, 10] combine_rows_per_src=[16286, 16316, 16378, 16378, 16350, 12180, 16374, 5142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108284 payload_rows_remote=95568 dup_rows=22788 dup_rows_remote=21302 dedup_ratio=0.173859 router_tokens_per_expert=[0, 3, 0, 3457, 3, 1104, 1, 0, 99, 2, 25, 4094, 8081, 8122, 8096, 89, 0, 1154, 13, 0, 5690, 5, 7286, 35, 1318, 8105, 1, 3, 1, 8169, 580, 0] payload_rows_per_dst=[16384, 16182, 8254, 16326, 12716, 15188, 6890, 16344] dup_rows_per_dst=[6852, 2206, 3874, 14, 1486, 7156, 44, 1156] combine_rows_per_src=[16384, 16182, 8254, 16326, 12716, 15188, 6890, 16344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118276 payload_rows_remote=101896 dup_rows=12796 dup_rows_remote=12716 dedup_ratio=0.097626 router_tokens_per_expert=[4968, 0, 0, 0, 0, 0, 2, 12, 0, 3, 908, 554, 3, 0, 42, 8185, 7279, 6221, 0, 0, 7101, 4846, 8192, 0, 0, 14, 0, 0, 0, 7921, 1094, 8191] payload_rows_per_dst=[16322, 16384, 13590, 16380, 14558, 8472, 16382, 16188] dup_rows_per_dst=[9036, 28, 1782, 80, 0, 0, 28, 1842] combine_rows_per_src=[16322, 16384, 13590, 16380, 14558, 8472, 16382, 16188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94032 payload_rows_remote=77654 dup_rows=37040 dup_rows_remote=26518 dedup_ratio=0.282593 router_tokens_per_expert=[291, 0, 0, 0, 0, 0, 150, 2510, 1, 278, 44, 0, 12, 9, 103, 8085, 4355, 8002, 1, 26, 6659, 8170, 8025, 0, 16, 184, 1140, 0, 189, 8144, 950, 8192] payload_rows_per_dst=[582, 10876, 11964, 10998, 14368, 16342, 12524, 16378] dup_rows_per_dst=[0, 300, 644, 226, 280, 22586, 2482, 10522] combine_rows_per_src=[582, 10876, 11964, 10998, 14368, 16342, 12524, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=791937024 dispatch_meta_bytes=458432 combine_payload_bytes=791937024 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=792395456 ingress_bytes=1144258560 link_bytes=1144258560 token_bytes=1584332480 weight_movement_bytes=352321536 total_nvlink_bytes=1936654016 weight_to_token_ratio=0.222379 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.407370299e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116528 payload_rows_remote=100164 dup_rows=14544 dup_rows_remote=10878 dedup_ratio=0.110962 router_tokens_per_expert=[0, 7654, 8161, 0, 58, 292, 2974, 8175, 252, 1, 6167, 82, 2872, 0, 15, 1, 10, 16, 0, 51, 6190, 4139, 25, 0, 3884, 3195, 2, 3137, 8182, 0, 0, 1] payload_rows_per_dst=[16364, 16364, 12816, 13956, 15410, 12408, 16346, 12864] dup_rows_per_dst=[3666, 6428, 188, 12, 52, 108, 4090, 0] combine_rows_per_src=[16364, 16364, 12816, 13956, 15410, 12408, 16346, 12864] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=811728896 dispatch_meta_bytes=458512 combine_payload_bytes=811728896 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=812187408 ingress_bytes=1164050432 link_bytes=1164050432 token_bytes=1623916304 weight_movement_bytes=352321536 total_nvlink_bytes=1976237840 weight_to_token_ratio=0.216958 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.518196521e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=782893056 dispatch_meta_bytes=467480 combine_payload_bytes=782893056 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=783360536 ingress_bytes=1135214592 link_bytes=1135214592 token_bytes=1566253592 weight_movement_bytes=352321536 total_nvlink_bytes=1918575128 weight_to_token_ratio=0.224945 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=25600 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.356727853e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122270 payload_rows_remote=106224 dup_rows=8802 dup_rows_remote=8620 dedup_ratio=0.067154 router_tokens_per_expert=[3672, 0, 0, 5, 0, 55, 0, 43, 0, 0, 101, 0, 46, 3, 5, 8183, 7975, 8160, 77, 0, 5887, 4880, 8192, 0, 0, 1309, 47, 0, 1, 8132, 740, 8023] payload_rows_per_dst=[16368, 16384, 16046, 16378, 16334, 9740, 16384, 14636] dup_rows_per_dst=[2780, 152, 182, 96, 1504, 0, 2712, 1376] combine_rows_per_src=[16368, 16384, 16046, 16378, 16334, 9740, 16384, 14636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=834732032 dispatch_meta_bytes=458448 combine_payload_bytes=834732032 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=835190480 ingress_bytes=1363214336 link_bytes=1363214336 token_bytes=1669922512 weight_movement_bytes=528482304 total_nvlink_bytes=2198404816 weight_to_token_ratio=0.316471 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=30080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.633431248e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=636141568 dispatch_meta_bytes=416688 combine_payload_bytes=636141568 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=636558256 ingress_bytes=1693106176 link_bytes=1693106176 token_bytes=1272699824 weight_movement_bytes=1056964608 total_nvlink_bytes=2329664432 weight_to_token_ratio=0.830490 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=24320 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.480687848e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=820543488 dispatch_meta_bytes=444168 combine_payload_bytes=820543488 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=820987656 ingress_bytes=1877508096 link_bytes=1877508096 token_bytes=1641531144 weight_movement_bytes=1056964608 total_nvlink_bytes=2698495752 weight_to_token_ratio=0.643889 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.051326163e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115134 payload_rows_remote=101462 dup_rows=15938 dup_rows_remote=8772 dedup_ratio=0.121597 router_tokens_per_expert=[6, 78, 0, 12, 0, 7, 1, 3966, 23, 0, 26, 0, 218, 0, 20, 7975, 5190, 8189, 12, 104, 6879, 8101, 8143, 19, 0, 4, 0, 0, 129, 8154, 95, 8185] payload_rows_per_dst=[9192, 16358, 16380, 16296, 10604, 13672, 16312, 16320] dup_rows_per_dst=[160, 7920, 96, 130, 30, 7166, 0, 436] combine_rows_per_src=[9192, 16358, 16380, 16296, 10604, 13672, 16312, 16320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=870187008 dispatch_meta_bytes=459376 combine_payload_bytes=870187008 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=870646384 ingress_bytes=1222508544 link_bytes=1222508544 token_bytes=1740833392 weight_movement_bytes=352321536 total_nvlink_bytes=2093154928 weight_to_token_ratio=0.202387 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.845537546e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114492 payload_rows_remote=99546 dup_rows=16580 dup_rows_remote=16226 dedup_ratio=0.126495 router_tokens_per_expert=[111, 8192, 7038, 22, 0, 0, 5249, 8123, 726, 0, 8044, 0, 3183, 0, 10, 0, 4, 0, 0, 0, 8186, 8164, 0, 105, 158, 29, 2, 1, 8189, 0, 0, 0] payload_rows_per_dst=[14298, 3758, 16104, 16296, 16384, 16328, 14946, 16378] dup_rows_per_dst=[44, 0, 3750, 6336, 8, 6088, 354, 0] combine_rows_per_src=[14298, 3758, 16104, 16296, 16384, 16328, 14946, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108136 payload_rows_remote=91758 dup_rows=22936 dup_rows_remote=6622 dedup_ratio=0.174988 router_tokens_per_expert=[2, 8177, 7233, 1, 0, 96, 8105, 8145, 134, 5, 5660, 0, 1320, 0, 0, 1, 67, 0, 12, 3, 8192, 1198, 0, 1, 1044, 4186, 903, 2861, 8177, 13, 0, 0] payload_rows_per_dst=[14472, 16378, 11590, 10284, 16360, 8202, 14494, 16356] dup_rows_per_dst=[0, 16314, 8, 550, 158, 2388, 3494, 24] combine_rows_per_src=[14472, 16378, 11590, 10284, 16360, 8202, 14494, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=831176704 dispatch_meta_bytes=440936 combine_payload_bytes=831176704 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=831617640 ingress_bytes=1888141312 link_bytes=1888141312 token_bytes=1662794344 weight_movement_bytes=1056964608 total_nvlink_bytes=2719758952 weight_to_token_ratio=0.635656 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=22528 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.057280320e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=815480832 dispatch_meta_bytes=463088 combine_payload_bytes=815480832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=815943920 ingress_bytes=1167802368 link_bytes=1167802368 token_bytes=1631424752 weight_movement_bytes=352321536 total_nvlink_bytes=1983746288 weight_to_token_ratio=0.215959 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=17024 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.539205796e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115506 payload_rows_remote=99588 dup_rows=15566 dup_rows_remote=9818 dedup_ratio=0.118759 router_tokens_per_expert=[6521, 0, 84, 530, 0, 0, 23, 103, 8172, 12, 3, 51, 2, 0, 0, 142, 3995, 6838, 8099, 0, 4, 0, 3, 5309, 8191, 3824, 0, 107, 8151, 3, 5359, 10] payload_rows_per_dst=[14046, 16202, 16374, 16380, 15918, 10642, 15210, 10734] dup_rows_per_dst=[224, 228, 102, 230, 5748, 1364, 7660, 10] combine_rows_per_src=[14046, 16202, 16374, 16380, 15918, 10642, 15210, 10734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=106908 payload_rows_remote=91518 dup_rows=24164 dup_rows_remote=17768 dedup_ratio=0.184357 router_tokens_per_expert=[173, 8191, 7971, 0, 0, 28, 4850, 8192, 352, 1, 4269, 0, 3199, 0, 0, 5, 8, 0, 0, 0, 7669, 8192, 0, 0, 13, 2433, 727, 15, 8191, 0, 1057, 0] payload_rows_per_dst=[15942, 9700, 9530, 15390, 16382, 16384, 7198, 16382] dup_rows_per_dst=[346, 7892, 70, 6396, 16, 6844, 486, 2114] combine_rows_per_src=[15942, 9700, 9530, 15390, 16382, 16384, 7198, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115940 payload_rows_remote=99662 dup_rows=15132 dup_rows_remote=15040 dedup_ratio=0.115448 router_tokens_per_expert=[29, 8112, 8172, 0, 3, 4, 4297, 8192, 0, 2, 7393, 84, 57, 0, 5, 0, 4, 6, 0, 0, 8190, 7565, 0, 45, 7, 4266, 0, 918, 8130, 0, 55, 0] payload_rows_per_dst=[14314, 5002, 14954, 16384, 16242, 16382, 16384, 16278] dup_rows_per_dst=[22, 2, 1018, 124, 2068, 9204, 2602, 92] combine_rows_per_src=[14314, 5002, 14954, 16384, 16242, 16382, 16384, 16278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[1, 0, 2, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110664 payload_rows_remote=95158 dup_rows=20408 dup_rows_remote=13432 dedup_ratio=0.155701 router_tokens_per_expert=[16, 7546, 3471, 0, 3, 3, 15, 592, 29, 4, 2, 1, 5819, 8186, 201, 5496, 0, 1526, 1, 8032, 18, 8019, 1, 3, 990, 0, 106, 2, 7127, 0, 8169, 158] payload_rows_per_dst=[15506, 16348, 16378, 13496, 16250, 16038, 2168, 14480] dup_rows_per_dst=[6976, 1216, 66, 9292, 2696, 44, 28, 90] combine_rows_per_src=[15506, 16348, 16378, 13496, 16250, 16038, 2168, 14480] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=751681536 dispatch_meta_bytes=393520 combine_payload_bytes=751681536 prefetch_bytes=352321536 reduce_bytes=1233125376 egress_bytes=752075056 ingress_bytes=2337128448 link_bytes=2337128448 token_bytes=1503756592 weight_movement_bytes=1585446912 total_nvlink_bytes=3089203504 weight_to_token_ratio=1.054324 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=20480 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.308694374e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107788 payload_rows_remote=92252 dup_rows=23284 dup_rows_remote=23284 dedup_ratio=0.177643 router_tokens_per_expert=[134, 8023, 8175, 0, 0, 0, 8187, 8190, 3552, 0, 4216, 0, 2, 0, 22, 0, 0, 0, 0, 0, 8116, 7608, 0, 2, 406, 440, 0, 185, 8185, 10, 83, 0] payload_rows_per_dst=[16350, 16384, 15536, 9086, 16046, 16050, 1952, 16384] dup_rows_per_dst=[268, 15372, 0, 0, 0, 7362, 110, 172] combine_rows_per_src=[16350, 16384, 15536, 9086, 16046, 16050, 1952, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=749715456 dispatch_meta_bytes=437144 combine_payload_bytes=749715456 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=750152600 ingress_bytes=1278197760 link_bytes=1278197760 token_bytes=1499868056 weight_movement_bytes=528482304 total_nvlink_bytes=2028350360 weight_to_token_ratio=0.352353 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.157373910e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=816431104 dispatch_meta_bytes=458808 combine_payload_bytes=816431104 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=816889912 ingress_bytes=1344913408 link_bytes=1344913408 token_bytes=1633321016 weight_movement_bytes=528482304 total_nvlink_bytes=2161803320 weight_to_token_ratio=0.323563 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.530953690e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=815824896 dispatch_meta_bytes=437624 combine_payload_bytes=815824896 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=816262520 ingress_bytes=2048950272 link_bytes=2048950272 token_bytes=1632087416 weight_movement_bytes=1233125376 total_nvlink_bytes=2865212792 weight_to_token_ratio=0.755551 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.147326625e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=779534336 dispatch_meta_bytes=434360 combine_payload_bytes=779534336 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=779968696 ingress_bytes=1308016640 link_bytes=1308016640 token_bytes=1559503032 weight_movement_bytes=528482304 total_nvlink_bytes=2087985336 weight_to_token_ratio=0.338879 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.324347191e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113044 payload_rows_remote=105762 dup_rows=18028 dup_rows_remote=18014 dedup_ratio=0.137543 router_tokens_per_expert=[336, 8177, 8106, 0, 91, 0, 7896, 8192, 1, 6, 5011, 17, 1493, 11, 2, 5, 0, 2, 0, 15, 8126, 3641, 7, 0, 0, 131, 8, 6013, 8181, 0, 68, 0] payload_rows_per_dst=[16212, 7796, 16378, 16280, 16354, 7282, 16360, 16382] dup_rows_per_dst=[672, 0, 1884, 3018, 34, 14, 12290, 116] combine_rows_per_src=[16212, 7796, 16378, 16280, 16354, 7282, 16360, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110988 payload_rows_remote=94604 dup_rows=20084 dup_rows_remote=4860 dedup_ratio=0.153229 router_tokens_per_expert=[0, 8177, 7, 0, 0, 4, 12, 76, 296, 82, 0, 0, 7760, 8148, 0, 3890, 2, 190, 0, 8182, 0, 8004, 0, 16, 8, 7992, 44, 0, 4350, 0, 8192, 104] payload_rows_per_dst=[16354, 13274, 16382, 7780, 8570, 16040, 16384, 16204] dup_rows_per_dst=[14, 184, 670, 0, 330, 0, 15224, 3662] combine_rows_per_src=[16354, 13274, 16382, 7780, 8570, 16040, 16384, 16204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=755728384 dispatch_meta_bytes=462144 combine_payload_bytes=755728384 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=756190528 ingress_bytes=1108049920 link_bytes=1108049920 token_bytes=1511918912 weight_movement_bytes=352321536 total_nvlink_bytes=1864240448 weight_to_token_ratio=0.233029 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.204617029e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=866402304 dispatch_meta_bytes=495104 combine_payload_bytes=866402304 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=866897408 ingress_bytes=1747206144 link_bytes=1747206144 token_bytes=1733299712 weight_movement_bytes=880803840 total_nvlink_bytes=2614103552 weight_to_token_ratio=0.508166 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.783625087e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115146 payload_rows_remote=98778 dup_rows=15926 dup_rows_remote=8088 dedup_ratio=0.121506 router_tokens_per_expert=[0, 8192, 215, 0, 0, 89, 0, 3851, 83, 45, 1, 51, 8148, 8190, 3, 3583, 158, 0, 21, 7740, 8, 8130, 36, 0, 23, 4696, 0, 0, 4103, 7, 8163, 0] payload_rows_per_dst=[16384, 16368, 16384, 16300, 15766, 16322, 9402, 8220] dup_rows_per_dst=[430, 7838, 356, 7168, 72, 26, 36, 0] combine_rows_per_src=[16384, 16368, 16384, 16300, 15766, 16322, 9402, 8220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=774995968 dispatch_meta_bytes=397856 combine_payload_bytes=774995968 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=775393824 ingress_bytes=1127317504 link_bytes=1127317504 token_bytes=1550389792 weight_movement_bytes=352321536 total_nvlink_bytes=1902711328 weight_to_token_ratio=0.227247 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=20352 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.312507457e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116684 payload_rows_remote=106508 dup_rows=14388 dup_rows_remote=9956 dedup_ratio=0.109772 router_tokens_per_expert=[0, 8008, 0, 0, 0, 14, 0, 278, 6, 0, 0, 0, 7991, 8192, 0, 5088, 9, 2, 253, 7979, 54, 8190, 0, 0, 0, 6858, 0, 0, 4422, 0, 8192, 0] payload_rows_per_dst=[16016, 16384, 16384, 10176, 16160, 16380, 16340, 8844] dup_rows_per_dst=[0, 584, 12, 4432, 326, 108, 8926, 0] combine_rows_per_src=[16016, 16384, 16384, 10176, 16160, 16380, 16340, 8844] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=101518 payload_rows_remote=85134 dup_rows=29554 dup_rows_remote=5408 dedup_ratio=0.225479 router_tokens_per_expert=[28, 7282, 326, 4, 58, 0, 3, 470, 332, 0, 0, 24, 7710, 8158, 0, 7962, 1, 131, 911, 8106, 0, 8183, 0, 0, 2, 5, 2, 8, 7647, 26, 8157, 0] payload_rows_per_dst=[11402, 948, 13660, 11750, 15572, 16366, 15436, 16384] dup_rows_per_dst=[646, 114, 654, 960, 1326, 1690, 18, 24146] combine_rows_per_src=[11402, 948, 13660, 11750, 15572, 16366, 15436, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119546 payload_rows_remote=108252 dup_rows=11526 dup_rows_remote=11252 dedup_ratio=0.087936 router_tokens_per_expert=[508, 1, 4178, 83, 0, 11, 8088, 7999, 0, 133, 0, 3438, 0, 5132, 521, 6, 8166, 0, 7642, 0, 0, 67, 1, 6, 7968, 5, 28, 432, 0, 3559, 22, 7542] payload_rows_per_dst=[11294, 16012, 16276, 12332, 16328, 16186, 16378, 14740] dup_rows_per_dst=[274, 2082, 6154, 1886, 504, 138, 488, 0] combine_rows_per_src=[11294, 16012, 16276, 12332, 16328, 16186, 16378, 14740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119280 payload_rows_remote=106914 dup_rows=11792 dup_rows_remote=11656 dedup_ratio=0.089966 router_tokens_per_expert=[609, 33, 4034, 1447, 15, 8055, 1, 20, 74, 0, 0, 658, 70, 8052, 6847, 0, 77, 4017, 1, 7995, 2, 6618, 8, 2, 8080, 172, 0, 449, 0, 11, 8175, 14] payload_rows_per_dst=[11882, 16166, 16310, 13800, 12366, 16018, 16364, 16374] dup_rows_per_dst=[364, 16, 1258, 34, 136, 8920, 1038, 26] combine_rows_per_src=[11882, 16166, 16310, 13800, 12366, 16018, 16364, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=809189376 dispatch_meta_bytes=427464 combine_payload_bytes=809189376 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=809616840 ingress_bytes=1161510912 link_bytes=1161510912 token_bytes=1618806216 weight_movement_bytes=352321536 total_nvlink_bytes=1971127752 weight_to_token_ratio=0.217643 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=27904 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.503976269e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112166 payload_rows_remote=95788 dup_rows=18906 dup_rows_remote=18860 dedup_ratio=0.144241 router_tokens_per_expert=[21, 7239, 0, 0, 18, 2, 0, 12, 19, 37, 0, 0, 7778, 8156, 0, 8139, 8, 903, 887, 8192, 0, 8174, 0, 9, 0, 99, 0, 0, 7674, 0, 8169, 0] payload_rows_per_dst=[14520, 16374, 16378, 16360, 16384, 16366, 436, 15348] dup_rows_per_dst=[0, 28, 46, 15212, 3596, 0, 24, 0] combine_rows_per_src=[14520, 16374, 16378, 16360, 16384, 16366, 436, 15348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=872513536 dispatch_meta_bytes=465856 combine_payload_bytes=872513536 prefetch_bytes=352321536 reduce_bytes=1233125376 egress_bytes=872979392 ingress_bytes=2457960448 link_bytes=2457960448 token_bytes=1745492928 weight_movement_bytes=1585446912 total_nvlink_bytes=3330939840 weight_to_token_ratio=0.908309 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=20224 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.376355251e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=697417728 dispatch_meta_bytes=362168 combine_payload_bytes=697417728 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=697779896 ingress_bytes=1402060800 link_bytes=1402060800 token_bytes=1395197624 weight_movement_bytes=704643072 total_nvlink_bytes=2099840696 weight_to_token_ratio=0.505049 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=27520 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.850955231e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=886800384 dispatch_meta_bytes=478016 combine_payload_bytes=886800384 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=887278400 ingress_bytes=1767604224 link_bytes=1767604224 token_bytes=1774078784 weight_movement_bytes=880803840 total_nvlink_bytes=2654882624 weight_to_token_ratio=0.496485 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=16512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.897845820e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=875839488 dispatch_meta_bytes=474280 combine_payload_bytes=875839488 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=876313768 ingress_bytes=2108964864 link_bytes=2108964864 token_bytes=1752153256 weight_movement_bytes=1233125376 total_nvlink_bytes=2985278632 weight_to_token_ratio=0.703777 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=18048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.180932291e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123286 payload_rows_remote=107372 dup_rows=7786 dup_rows_remote=7698 dedup_ratio=0.059402 router_tokens_per_expert=[10, 8181, 55, 2, 0, 1, 0, 0, 156, 0, 4, 149, 8016, 8139, 28, 5050, 123, 3, 338, 8190, 0, 7954, 45, 2, 126, 3072, 1, 3, 7727, 0, 8161, 0] payload_rows_per_dst=[16384, 16324, 16296, 10156, 16384, 15914, 16374, 15454] dup_rows_per_dst=[112, 0, 600, 0, 924, 88, 6062, 0] combine_rows_per_src=[16384, 16324, 16296, 10156, 16384, 15914, 16374, 15454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112932 payload_rows_remote=96594 dup_rows=18140 dup_rows_remote=18022 dedup_ratio=0.138397 router_tokens_per_expert=[4173, 0, 4531, 0, 0, 3582, 8178, 8155, 0, 1, 0, 12, 1, 8149, 0, 22, 8160, 0, 8067, 0, 165, 140, 12, 0, 4086, 0, 23, 0, 0, 69, 0, 8010] payload_rows_per_dst=[8938, 15340, 7408, 16370, 16134, 16384, 16338, 16020] dup_rows_per_dst=[1040, 4910, 2, 7296, 806, 3830, 118, 138] combine_rows_per_src=[8938, 15340, 7408, 16370, 16134, 16384, 16338, 16020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=784695296 dispatch_meta_bytes=458592 combine_payload_bytes=784695296 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=785153888 ingress_bytes=1137016832 link_bytes=1137016832 token_bytes=1569849184 weight_movement_bytes=352321536 total_nvlink_bytes=1922170720 weight_to_token_ratio=0.224430 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.366819645e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115116 payload_rows_remote=99090 dup_rows=15956 dup_rows_remote=14022 dedup_ratio=0.121735 router_tokens_per_expert=[124, 0, 7787, 0, 0, 1399, 8180, 7581, 81, 5, 0, 4996, 11, 6516, 27, 0, 7900, 0, 8174, 0, 41, 3, 220, 0, 4101, 3, 0, 161, 40, 0, 0, 8186] payload_rows_per_dst=[15604, 16026, 16372, 14188, 15794, 16370, 8522, 12240] dup_rows_per_dst=[218, 1934, 10146, 2064, 988, 518, 8, 80] combine_rows_per_src=[15604, 16026, 16372, 14188, 15794, 16370, 8522, 12240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=879591424 dispatch_meta_bytes=460280 combine_payload_bytes=879591424 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=880051704 ingress_bytes=1231912960 link_bytes=1231912960 token_bytes=1759643128 weight_movement_bytes=352321536 total_nvlink_bytes=2111964664 weight_to_token_ratio=0.200223 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.898198350e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=791298048 dispatch_meta_bytes=458464 combine_payload_bytes=791298048 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=791756512 ingress_bytes=1143619584 link_bytes=1143619584 token_bytes=1583054560 weight_movement_bytes=352321536 total_nvlink_bytes=1935376096 weight_to_token_ratio=0.222558 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=29056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.403792300e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117668 payload_rows_remote=101424 dup_rows=13404 dup_rows_remote=13404 dedup_ratio=0.102264 router_tokens_per_expert=[4179, 74, 7785, 165, 0, 906, 8192, 4301, 4, 6, 0, 3755, 0, 8122, 0, 0, 6546, 0, 4669, 0, 0, 0, 6, 296, 8169, 0, 0, 36, 0, 133, 0, 8192] payload_rows_per_dst=[16036, 10570, 16300, 16244, 9394, 16384, 16356, 16384] dup_rows_per_dst=[8148, 66, 4266, 0, 0, 604, 54, 266] combine_rows_per_src=[16036, 10570, 16300, 16244, 9394, 16384, 16356, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110750 payload_rows_remote=94516 dup_rows=20322 dup_rows_remote=20292 dedup_ratio=0.155045 router_tokens_per_expert=[7566, 0, 4052, 0, 0, 985, 7635, 4182, 0, 58, 3, 3341, 0, 8119, 1, 0, 8150, 0, 8178, 92, 0, 623, 111, 0, 4246, 0, 62, 0, 15, 0, 15, 8102] payload_rows_per_dst=[15150, 8274, 6758, 16238, 16384, 15472, 16240, 16234] dup_rows_per_dst=[1128, 942, 46, 2, 8340, 2384, 7450, 30] combine_rows_per_src=[15150, 8274, 6758, 16238, 16384, 15472, 16240, 16234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=811745280 dispatch_meta_bytes=452448 combine_payload_bytes=811745280 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=812197728 ingress_bytes=1868709888 link_bytes=1868709888 token_bytes=1623943008 weight_movement_bytes=1056964608 total_nvlink_bytes=2680907616 weight_to_token_ratio=0.650863 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=28160 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.046399533e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=87704 payload_rows_remote=71336 dup_rows=43368 dup_rows_remote=39892 dedup_ratio=0.330872 router_tokens_per_expert=[7808, 257, 0, 2, 1742, 2, 6, 8143, 7886, 84, 4193, 568, 712, 1, 3480, 1551, 12, 35, 1156, 4044, 2953, 4098, 0, 0, 0, 6341, 1275, 4, 1066, 0, 0, 8117] payload_rows_per_dst=[16368, 16384, 8602, 12936, 8296, 13978, 9008, 2132] dup_rows_per_dst=[3476, 15792, 4956, 4784, 2198, 12162, 0, 0] combine_rows_per_src=[16368, 16384, 8602, 12936, 8296, 13978, 9008, 2132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118108 payload_rows_remote=101740 dup_rows=12964 dup_rows_remote=10308 dedup_ratio=0.098907 router_tokens_per_expert=[7376, 14, 17, 0, 8178, 7496, 293, 4, 17, 0, 159, 8, 3131, 0, 8164, 256, 1321, 22, 4, 1, 998, 134, 8052, 8085, 9, 178, 3, 0, 0, 898, 2554, 8164] payload_rows_per_dst=[14786, 15008, 16290, 16362, 16368, 16364, 16364, 6566] dup_rows_per_dst=[28, 588, 258, 6740, 2656, 2004, 352, 338] combine_rows_per_src=[14786, 15008, 16290, 16362, 16368, 16364, 16364, 6566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119934 payload_rows_remote=103554 dup_rows=11138 dup_rows_remote=5488 dedup_ratio=0.084976 router_tokens_per_expert=[4194, 247, 6322, 52, 0, 96, 5211, 8122, 0, 1, 30, 4052, 0, 8025, 0, 4, 7166, 4, 6934, 0, 0, 170, 0, 1, 6689, 0, 0, 24, 0, 0, 0, 8192] payload_rows_per_dst=[16382, 16248, 16380, 16058, 14344, 10738, 13400, 16384] dup_rows_per_dst=[5248, 188, 5650, 0, 0, 26, 26, 0] combine_rows_per_src=[16382, 16248, 16380, 16058, 14344, 10738, 13400, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=830865408 dispatch_meta_bytes=459312 combine_payload_bytes=830865408 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=831324720 ingress_bytes=1183186944 link_bytes=1183186944 token_bytes=1662190128 weight_movement_bytes=352321536 total_nvlink_bytes=2014511664 weight_to_token_ratio=0.211962 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=28800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.625353000e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=774275072 dispatch_meta_bytes=459232 combine_payload_bytes=774275072 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=774734304 ingress_bytes=1831239680 link_bytes=1831239680 token_bytes=1549009376 weight_movement_bytes=1056964608 total_nvlink_bytes=2605973984 weight_to_token_ratio=0.682349 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.025417781e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=584384512 dispatch_meta_bytes=444912 combine_payload_bytes=584384512 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=584829424 ingress_bytes=1112866816 link_bytes=1112866816 token_bytes=1169213936 weight_movement_bytes=528482304 total_nvlink_bytes=1697696240 weight_to_token_ratio=0.451998 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.231589636e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120786 payload_rows_remote=105022 dup_rows=10286 dup_rows_remote=10158 dedup_ratio=0.078476 router_tokens_per_expert=[3702, 96, 4602, 11, 0, 0, 7869, 8074, 490, 0, 1, 627, 257, 8023, 3, 0, 7762, 0, 7666, 95, 0, 70, 1, 6, 7367, 0, 1, 135, 205, 337, 22, 8114] payload_rows_per_dst=[15648, 16148, 9598, 16318, 15354, 15764, 15572, 16384] dup_rows_per_dst=[34, 0, 830, 1388, 168, 128, 6766, 972] combine_rows_per_src=[15648, 16148, 9598, 16318, 15354, 15764, 15572, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=848314368 dispatch_meta_bytes=436168 combine_payload_bytes=848314368 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=848750536 ingress_bytes=1200635904 link_bytes=1200635904 token_bytes=1697064904 weight_movement_bytes=352321536 total_nvlink_bytes=2049386440 weight_to_token_ratio=0.207606 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=26624 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.723059892e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108912 payload_rows_remote=92716 dup_rows=22160 dup_rows_remote=10108 dedup_ratio=0.169067 router_tokens_per_expert=[8165, 0, 0, 0, 7694, 81, 0, 8189, 4107, 0, 946, 4186, 2, 0, 0, 51, 0, 3, 0, 2426, 2544, 1205, 3, 1, 0, 6026, 8097, 0, 3612, 232, 25, 7941] payload_rows_per_dst=[16330, 15388, 9398, 16384, 13040, 5828, 16196, 16348] dup_rows_per_dst=[0, 162, 888, 100, 8, 1678, 12052, 7272] combine_rows_per_src=[16330, 15388, 9398, 16384, 13040, 5828, 16196, 16348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=833454080 dispatch_meta_bytes=448192 combine_payload_bytes=833454080 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=833902272 ingress_bytes=1185775616 link_bytes=1185775616 token_bytes=1667356352 weight_movement_bytes=352321536 total_nvlink_bytes=2019677888 weight_to_token_ratio=0.211305 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.639848482e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=103260 payload_rows_remote=86894 dup_rows=27812 dup_rows_remote=18992 dedup_ratio=0.212189 router_tokens_per_expert=[8014, 105, 154, 1, 3824, 3, 51, 8178, 8033, 75, 2181, 875, 0, 0, 976, 0, 0, 709, 1, 5557, 0, 1588, 1, 1465, 13, 3111, 8171, 228, 4490, 20, 0, 7712] payload_rows_per_dst=[16328, 16366, 15702, 8192, 15390, 15580, 6682, 9020] dup_rows_per_dst=[220, 8820, 5552, 1952, 5294, 5952, 22, 0] combine_rows_per_src=[16328, 16366, 15702, 8192, 15390, 15580, 6682, 9020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=860340224 dispatch_meta_bytes=460720 combine_payload_bytes=860340224 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=860800944 ingress_bytes=1564983296 link_bytes=1564983296 token_bytes=1721141168 weight_movement_bytes=704643072 total_nvlink_bytes=2425784240 weight_to_token_ratio=0.409405 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.017045 gemm_active_groups=14 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.763253201e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=102068 payload_rows_remote=85736 dup_rows=29004 dup_rows_remote=26588 dedup_ratio=0.221283 router_tokens_per_expert=[8192, 0, 0, 0, 7455, 0, 861, 8185, 6137, 0, 4817, 72, 75, 297, 845, 0, 0, 88, 0, 3299, 2078, 2749, 0, 0, 0, 64, 8164, 0, 3999, 0, 0, 8159] payload_rows_per_dst=[16384, 15350, 16184, 16332, 16378, 9868, 128, 11444] dup_rows_per_dst=[0, 2726, 5868, 2416, 5336, 5682, 0, 6976] combine_rows_per_src=[16384, 15350, 16184, 16332, 16378, 9868, 128, 11444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=759529472 dispatch_meta_bytes=411296 combine_payload_bytes=759529472 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=759940768 ingress_bytes=1640333312 link_bytes=1640333312 token_bytes=1519470240 weight_movement_bytes=880803840 total_nvlink_bytes=2400274080 weight_to_token_ratio=0.579678 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.185181838e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94466 payload_rows_remote=78082 dup_rows=36606 dup_rows_remote=28240 dedup_ratio=0.279282 router_tokens_per_expert=[8190, 39, 2195, 62, 2680, 0, 13, 8190, 7592, 32, 162, 1328, 0, 19, 4788, 0, 0, 2084, 0, 2239, 731, 3304, 0, 0, 8, 5435, 4813, 70, 3369, 0, 1, 8192] payload_rows_per_dst=[12558, 5386, 6988, 16370, 12370, 8026, 16384, 16384] dup_rows_per_dst=[182, 0, 1488, 9466, 4464, 44, 12596, 8366] combine_rows_per_src=[12558, 5386, 6988, 16370, 12370, 8026, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=104040 payload_rows_remote=87658 dup_rows=27032 dup_rows_remote=26862 dedup_ratio=0.206238 router_tokens_per_expert=[47, 5, 32, 3, 0, 0, 5, 168, 14, 1957, 61, 8178, 405, 13, 11, 3687, 8190, 7993, 8175, 0, 7400, 1, 273, 7564, 6637, 0, 3, 24, 4336, 354, 0, 0] payload_rows_per_dst=[16382, 16372, 16376, 16384, 15986, 7854, 13978, 708] dup_rows_per_dst=[170, 326, 18150, 8214, 0, 0, 172, 0] combine_rows_per_src=[16382, 16372, 16376, 16384, 15986, 7854, 13978, 708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=711835648 dispatch_meta_bytes=423544 combine_payload_bytes=711835648 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=712259192 ingress_bytes=1416478720 link_bytes=1416478720 token_bytes=1424094840 weight_movement_bytes=704643072 total_nvlink_bytes=2128737912 weight_to_token_ratio=0.494801 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=17024 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.931689565e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=102852 payload_rows_remote=87228 dup_rows=28220 dup_rows_remote=18460 dedup_ratio=0.215302 router_tokens_per_expert=[8180, 52, 0, 0, 1725, 0, 2615, 6599, 7791, 0, 4689, 212, 14, 0, 1894, 0, 0, 90, 0, 1229, 3629, 2138, 0, 0, 0, 8191, 4114, 0, 4270, 6, 2, 8096] payload_rows_per_dst=[16384, 14878, 15624, 12220, 2498, 16314, 16382, 8552] dup_rows_per_dst=[80, 2996, 9760, 3792, 176, 11412, 0, 4] combine_rows_per_src=[16384, 14878, 15624, 12220, 2498, 16314, 16382, 8552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=702349312 dispatch_meta_bytes=449296 combine_payload_bytes=702349312 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=702798608 ingress_bytes=1230831616 link_bytes=1230831616 token_bytes=1405147920 weight_movement_bytes=528482304 total_nvlink_bytes=1933630224 weight_to_token_ratio=0.376104 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.018939 gemm_active_groups=13 gemm_peak_group_rows=26368 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.892143275e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113920 payload_rows_remote=105068 dup_rows=17152 dup_rows_remote=17152 dedup_ratio=0.130859 router_tokens_per_expert=[4278, 7, 8, 0, 0, 214, 0, 5, 9, 1, 128, 460, 63, 0, 351, 8188, 4383, 7964, 0, 43, 3866, 7322, 7839, 0, 18, 601, 0, 0, 0, 7896, 3743, 8149] payload_rows_per_dst=[16226, 15690, 16384, 16384, 8852, 8180, 16344, 15860] dup_rows_per_dst=[6556, 10, 1146, 820, 0, 0, 1202, 7418] combine_rows_per_src=[16226, 15690, 16384, 16384, 8852, 8180, 16344, 15860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=639647744 dispatch_meta_bytes=425288 combine_payload_bytes=639647744 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=640073032 ingress_bytes=1696612352 link_bytes=1696612352 token_bytes=1279720776 weight_movement_bytes=1056964608 total_nvlink_bytes=2336685384 weight_to_token_ratio=0.825934 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=19456 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.500320970e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=718094336 dispatch_meta_bytes=458080 combine_payload_bytes=718094336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=718552416 ingress_bytes=1070415872 link_bytes=1070415872 token_bytes=1436646752 weight_movement_bytes=352321536 total_nvlink_bytes=1788968288 weight_to_token_ratio=0.245239 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.993882070e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107992 payload_rows_remote=103494 dup_rows=23080 dup_rows_remote=23042 dedup_ratio=0.176086 router_tokens_per_expert=[8154, 2, 0, 19, 3890, 107, 269, 8163, 7197, 4, 1426, 161, 0, 0, 68, 8, 0, 13, 15, 7223, 905, 1344, 3, 16, 0, 8083, 8062, 24, 2041, 199, 2, 8138] payload_rows_per_dst=[16348, 8360, 14248, 16224, 15548, 4498, 16382, 16384] dup_rows_per_dst=[2, 172, 2710, 38, 7102, 38, 8642, 4376] combine_rows_per_src=[16348, 8360, 14248, 16224, 15548, 4498, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=714571776 dispatch_meta_bytes=422752 combine_payload_bytes=714571776 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=714994528 ingress_bytes=1595375616 link_bytes=1595375616 token_bytes=1429566304 weight_movement_bytes=880803840 total_nvlink_bytes=2310370144 weight_to_token_ratio=0.616134 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=17920 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.933437507e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121748 payload_rows_remote=105364 dup_rows=9324 dup_rows_remote=8660 dedup_ratio=0.071136 router_tokens_per_expert=[11, 0, 1, 0, 0, 0, 10, 0, 0, 7938, 7, 8182, 147, 0, 0, 1, 8189, 8192, 8187, 0, 8118, 0, 0, 8180, 277, 0, 0, 65, 8017, 0, 0, 14] payload_rows_per_dst=[16384, 16378, 15878, 8430, 16384, 15876, 16384, 16034] dup_rows_per_dst=[14, 20, 12, 58, 0, 8044, 664, 512] combine_rows_per_src=[16384, 16378, 15878, 8430, 16384, 15876, 16384, 16034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=860717056 dispatch_meta_bytes=488880 combine_payload_bytes=860717056 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=861205936 ingress_bytes=1741520896 link_bytes=1741520896 token_bytes=1721922992 weight_movement_bytes=880803840 total_nvlink_bytes=2602726832 weight_to_token_ratio=0.511523 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=24832 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.751790071e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112962 payload_rows_remote=96608 dup_rows=18110 dup_rows_remote=17342 dedup_ratio=0.138168 router_tokens_per_expert=[361, 38, 28, 14, 76, 0, 0, 362, 0, 3739, 0, 8020, 244, 27, 0, 299, 8167, 8087, 8138, 0, 7745, 0, 0, 8047, 3858, 5, 5, 72, 8169, 0, 0, 35] payload_rows_per_dst=[16380, 16354, 16328, 16384, 16174, 15478, 15782, 82] dup_rows_per_dst=[866, 768, 15540, 924, 0, 0, 12, 0] combine_rows_per_src=[16380, 16354, 16328, 16384, 16174, 15478, 15782, 82] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=847822848 dispatch_meta_bytes=506144 combine_payload_bytes=847822848 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=848328992 ingress_bytes=1200144384 link_bytes=1200144384 token_bytes=1696151840 weight_movement_bytes=352321536 total_nvlink_bytes=2048473376 weight_to_token_ratio=0.207718 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=12928 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.720307585e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119702 payload_rows_remote=103320 dup_rows=11370 dup_rows_remote=4270 dedup_ratio=0.086746 router_tokens_per_expert=[14, 0, 0, 0, 0, 0, 302, 0, 10, 4083, 0, 8174, 3547, 0, 3, 0, 8191, 8192, 8180, 0, 8191, 0, 3, 8192, 104, 0, 0, 0, 7542, 1, 0, 807] payload_rows_per_dst=[16384, 16360, 8218, 16382, 16384, 13914, 16352, 15708] dup_rows_per_dst=[26, 604, 2438, 7100, 0, 6, 204, 992] combine_rows_per_src=[16384, 16360, 8218, 16382, 16384, 13914, 16352, 15708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=863141888 dispatch_meta_bytes=456096 combine_payload_bytes=863141888 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=863597984 ingress_bytes=1215463424 link_bytes=1215463424 token_bytes=1726739872 weight_movement_bytes=352321536 total_nvlink_bytes=2079061408 weight_to_token_ratio=0.204039 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=28800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.806087815e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96854 payload_rows_remote=80526 dup_rows=34218 dup_rows_remote=18954 dedup_ratio=0.261063 router_tokens_per_expert=[0, 73, 58, 0, 0, 3, 1196, 1163, 0, 3293, 2, 8163, 11, 15, 10, 244, 8187, 8139, 8189, 0, 8021, 34, 0, 8015, 246, 0, 0, 0, 7869, 2428, 28, 149] payload_rows_per_dst=[14928, 9634, 6590, 1162, 16218, 15668, 16326, 16328] dup_rows_per_dst=[262, 4658, 0, 56, 1764, 11722, 492, 15264] combine_rows_per_src=[14928, 9634, 6590, 1162, 16218, 15668, 16326, 16328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=91980 payload_rows_remote=75650 dup_rows=39092 dup_rows_remote=35524 dedup_ratio=0.298248 router_tokens_per_expert=[9, 8140, 5101, 4072, 4720, 74, 8145, 7593, 0, 2676, 13, 5, 0, 0, 38, 0, 16, 338, 0, 1196, 2573, 8186, 8, 0, 0, 8145, 225, 3, 29, 4213, 18, 0] payload_rows_per_dst=[16330, 16384, 15404, 15364, 10428, 8356, 8242, 1472] dup_rows_per_dst=[3568, 30634, 334, 40, 854, 3300, 334, 28] combine_rows_per_src=[16330, 16384, 15404, 15364, 10428, 8356, 8242, 1472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=791412736 dispatch_meta_bytes=455800 combine_payload_bytes=791412736 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=791868536 ingress_bytes=1143734272 link_bytes=1143734272 token_bytes=1583281272 weight_movement_bytes=352321536 total_nvlink_bytes=1935602808 weight_to_token_ratio=0.222526 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.404434505e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=846397440 dispatch_meta_bytes=430360 combine_payload_bytes=846397440 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=846827800 ingress_bytes=1198718976 link_bytes=1198718976 token_bytes=1693225240 weight_movement_bytes=352321536 total_nvlink_bytes=2045546776 weight_to_token_ratio=0.208077 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=28928 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.712325896e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116862 payload_rows_remote=100660 dup_rows=14210 dup_rows_remote=8734 dedup_ratio=0.108414 router_tokens_per_expert=[88, 45, 24, 0, 24, 0, 1, 280, 0, 3737, 54, 8088, 25, 0, 50, 4, 8168, 8149, 8164, 0, 8192, 0, 60, 8116, 352, 0, 0, 855, 8128, 2928, 0, 4] payload_rows_per_dst=[16384, 16384, 16202, 16384, 16298, 8312, 15242, 11656] dup_rows_per_dst=[258, 562, 5476, 158, 0, 0, 1930, 5826] combine_rows_per_src=[16384, 16384, 16202, 16384, 16298, 8312, 15242, 11656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 3, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=659668992 dispatch_meta_bytes=397920 combine_payload_bytes=659668992 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=660066912 ingress_bytes=1364312064 link_bytes=1364312064 token_bytes=1319735904 weight_movement_bytes=704643072 total_nvlink_bytes=2024378976 weight_to_token_ratio=0.533927 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=24320 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.639578067e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=619724800 dispatch_meta_bytes=444696 combine_payload_bytes=619724800 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=620169496 ingress_bytes=1852850176 link_bytes=1852850176 token_bytes=1239894296 weight_movement_bytes=1233125376 total_nvlink_bytes=2473019672 weight_to_token_ratio=0.994541 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=20864 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.037518757e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107340 payload_rows_remote=91794 dup_rows=23732 dup_rows_remote=23720 dedup_ratio=0.181061 router_tokens_per_expert=[636, 7728, 8110, 305, 12, 2, 5589, 8108, 3883, 0, 5702, 26, 288, 0, 0, 0, 36, 11, 4, 0, 8192, 3014, 0, 0, 5, 434, 493, 4193, 8129, 5, 631, 0] payload_rows_per_dst=[16274, 9670, 16240, 16384, 15546, 6028, 10928, 16270] dup_rows_per_dst=[1828, 12, 11006, 576, 12, 0, 9038, 1260] combine_rows_per_src=[16274, 9670, 16240, 16384, 15546, 6028, 10928, 16270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113588 payload_rows_remote=97238 dup_rows=17484 dup_rows_remote=15684 dedup_ratio=0.133392 router_tokens_per_expert=[0, 13, 0, 0, 0, 7, 0, 8, 1, 704, 212, 7977, 0, 154, 5, 0, 8014, 8178, 8025, 11, 8134, 0, 89, 8176, 12, 0, 0, 1, 8170, 7644, 0, 1] payload_rows_per_dst=[16094, 16014, 1758, 14648, 16374, 16350, 15966, 16384] dup_rows_per_dst=[24, 2, 76, 318, 4, 1800, 14, 15246] combine_rows_per_src=[16094, 16014, 1758, 14648, 16374, 16350, 15966, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=824606720 dispatch_meta_bytes=437576 combine_payload_bytes=824606720 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=825044296 ingress_bytes=1529249792 link_bytes=1529249792 token_bytes=1649651016 weight_movement_bytes=704643072 total_nvlink_bytes=2354294088 weight_to_token_ratio=0.427147 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=18432 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.563160495e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110122 payload_rows_remote=93738 dup_rows=20950 dup_rows_remote=20890 dedup_ratio=0.159836 router_tokens_per_expert=[0, 8172, 4261, 8159, 15, 0, 8191, 161, 0, 262, 0, 0, 0, 0, 0, 0, 0, 1545, 0, 413, 5648, 8133, 0, 134, 5, 8192, 25, 0, 4091, 7862, 266, 1] payload_rows_per_dst=[8522, 8222, 16384, 16278, 12066, 16374, 16384, 15892] dup_rows_per_dst=[0, 322, 522, 0, 42, 11456, 60, 8548] combine_rows_per_src=[8522, 8222, 16384, 16278, 12066, 16374, 16384, 15892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=100614 payload_rows_remote=90850 dup_rows=30458 dup_rows_remote=30394 dedup_ratio=0.232376 router_tokens_per_expert=[0, 8174, 7424, 8177, 15, 0, 7820, 19, 30, 0, 8, 0, 0, 21, 3, 4, 0, 0, 375, 3973, 1006, 8190, 104, 83, 2, 8180, 441, 0, 4097, 7322, 63, 5] payload_rows_per_dst=[15770, 9764, 16252, 16360, 8204, 2252, 15636, 16376] dup_rows_per_dst=[21288, 64, 68, 48, 8684, 134, 38, 134] combine_rows_per_src=[15770, 9764, 16252, 16360, 8204, 2252, 15636, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=751976448 dispatch_meta_bytes=462056 combine_payload_bytes=751976448 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=752438504 ingress_bytes=1456619520 link_bytes=1456619520 token_bytes=1504414952 weight_movement_bytes=704643072 total_nvlink_bytes=2209058024 weight_to_token_ratio=0.468383 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.017045 gemm_active_groups=13 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.156461289e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113474 payload_rows_remote=97098 dup_rows=17598 dup_rows_remote=17428 dedup_ratio=0.134262 router_tokens_per_expert=[0, 8192, 4840, 8081, 2484, 0, 8190, 2873, 0, 0, 0, 0, 0, 0, 192, 0, 0, 113, 0, 88, 2979, 8156, 0, 0, 0, 8176, 2815, 0, 3557, 4795, 5, 0] payload_rows_per_dst=[12868, 12920, 16380, 16376, 16384, 9778, 16352, 12416] dup_rows_per_dst=[540, 2258, 0, 170, 378, 6610, 5638, 2004] combine_rows_per_src=[12868, 12920, 16380, 16376, 16384, 9778, 16352, 12416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=796573696 dispatch_meta_bytes=451688 combine_payload_bytes=796573696 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=797025384 ingress_bytes=1853538304 link_bytes=1853538304 token_bytes=1593599080 weight_movement_bytes=1056964608 total_nvlink_bytes=2650563688 weight_to_token_ratio=0.663256 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=27008 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.037904080e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=767901696 dispatch_meta_bytes=458512 combine_payload_bytes=767901696 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=768360208 ingress_bytes=1296384000 link_bytes=1296384000 token_bytes=1536261904 weight_movement_bytes=528482304 total_nvlink_bytes=2064744208 weight_to_token_ratio=0.344005 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.259209263e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=106128 payload_rows_remote=89788 dup_rows=24944 dup_rows_remote=19564 dedup_ratio=0.190308 router_tokens_per_expert=[18, 1749, 8071, 60, 4939, 407, 0, 8131, 4424, 0, 1162, 0, 8, 10, 0, 81, 8124, 28, 7, 44, 164, 29, 44, 0, 7787, 6, 446, 0, 15, 3630, 8120, 8032] payload_rows_per_dst=[16340, 16374, 11172, 16342, 16306, 418, 16322, 12854] dup_rows_per_dst=[5380, 16820, 0, 96, 100, 56, 156, 2336] combine_rows_per_src=[16340, 16374, 11172, 16342, 16306, 418, 16322, 12854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98368 payload_rows_remote=81986 dup_rows=32704 dup_rows_remote=21770 dedup_ratio=0.249512 router_tokens_per_expert=[36, 8157, 2728, 8030, 654, 4, 8170, 936, 148, 0, 9, 0, 3, 0, 0, 15, 2, 4004, 0, 55, 4032, 8178, 0, 0, 1, 8190, 3, 0, 4171, 8007, 0, 3] payload_rows_per_dst=[3168, 8294, 16374, 16114, 8252, 16356, 13428, 16382] dup_rows_per_dst=[2, 3060, 296, 2362, 7980, 8064, 6, 10934] combine_rows_per_src=[3168, 8294, 16374, 16114, 8252, 16356, 13428, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=744243200 dispatch_meta_bytes=484976 combine_payload_bytes=744243200 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=744728176 ingress_bytes=1448886272 link_bytes=1448886272 token_bytes=1488971376 weight_movement_bytes=704643072 total_nvlink_bytes=2193614448 weight_to_token_ratio=0.473242 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=18560 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.113158328e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=795426816 dispatch_meta_bytes=458104 combine_payload_bytes=795426816 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=795884920 ingress_bytes=1323909120 link_bytes=1323909120 token_bytes=1591311736 weight_movement_bytes=528482304 total_nvlink_bytes=2119794040 weight_to_token_ratio=0.332105 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.413338445e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=109064 payload_rows_remote=92802 dup_rows=22008 dup_rows_remote=22008 dedup_ratio=0.167908 router_tokens_per_expert=[0, 8169, 7703, 7397, 3577, 0, 8098, 354, 0, 29, 0, 1, 0, 0, 131, 0, 0, 0, 0, 5, 5642, 8192, 0, 0, 89, 8191, 984, 0, 41, 6859, 11, 63] payload_rows_per_dst=[16380, 8178, 16262, 14814, 13788, 15346, 16384, 7912] dup_rows_per_dst=[14924, 2884, 0, 216, 10, 1744, 2144, 86] combine_rows_per_src=[16380, 8178, 16262, 14814, 13788, 15346, 16384, 7912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=735543296 dispatch_meta_bytes=437408 combine_payload_bytes=735543296 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=735980704 ingress_bytes=1616347136 link_bytes=1616347136 token_bytes=1471524000 weight_movement_bytes=880803840 total_nvlink_bytes=2352327840 weight_to_token_ratio=0.598566 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=30080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.050869265e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=671629312 dispatch_meta_bytes=415024 combine_payload_bytes=671629312 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=672044336 ingress_bytes=1552433152 link_bytes=1552433152 token_bytes=1343673648 weight_movement_bytes=880803840 total_nvlink_bytes=2224477488 weight_to_token_ratio=0.655519 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=23168 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.692977634e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=760233984 dispatch_meta_bytes=459240 combine_payload_bytes=760233984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=760693224 ingress_bytes=1112555520 link_bytes=1112555520 token_bytes=1520927208 weight_movement_bytes=352321536 total_nvlink_bytes=1873248744 weight_to_token_ratio=0.231649 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.229846508e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113030 payload_rows_remote=96692 dup_rows=18042 dup_rows_remote=17664 dedup_ratio=0.137650 router_tokens_per_expert=[27, 7203, 934, 0, 57, 0, 0, 4351, 215, 0, 0, 10, 7851, 8178, 37, 3958, 1, 211, 5, 8141, 49, 7768, 28, 0, 0, 4170, 75, 11, 4109, 9, 8136, 2] payload_rows_per_dst=[15664, 16384, 16378, 7978, 16338, 15686, 16366, 8236] dup_rows_per_dst=[664, 8704, 428, 12, 378, 4, 7848, 4] combine_rows_per_src=[15664, 16384, 16378, 7978, 16338, 15686, 16366, 8236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=104836 payload_rows_remote=89944 dup_rows=26236 dup_rows_remote=18150 dedup_ratio=0.200165 router_tokens_per_expert=[0, 8068, 4014, 7778, 3, 1, 8025, 887, 0, 25, 6, 159, 0, 19, 568, 0, 2, 4110, 110, 3214, 4095, 7300, 94, 0, 23, 7633, 345, 12, 4096, 4903, 46, 0] payload_rows_per_dst=[8028, 9884, 16350, 15698, 14622, 14892, 15334, 10028] dup_rows_per_dst=[0, 90, 80, 784, 8442, 8086, 692, 8062] combine_rows_per_src=[8028, 9884, 16350, 15698, 14622, 14892, 15334, 10028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119588 payload_rows_remote=103334 dup_rows=11484 dup_rows_remote=11478 dedup_ratio=0.087616 router_tokens_per_expert=[0, 845, 8097, 0, 3987, 1248, 2, 8126, 8189, 0, 0, 1, 30, 0, 31, 5, 8192, 0, 0, 0, 1901, 0, 0, 89, 8103, 0, 27, 0, 75, 257, 8153, 8178] payload_rows_per_dst=[10970, 10470, 16378, 16382, 16384, 16376, 16254, 16374] dup_rows_per_dst=[20, 4, 4534, 56, 0, 3856, 6, 3008] combine_rows_per_src=[10970, 10470, 16378, 16382, 16384, 16376, 16254, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=23416195760 prefetch_bytes=5108662272 reduce_bytes=4756340736 weight_movement_bytes=9865003008 egress_bytes=11711630000 ingress_bytes=21569568768 link_bytes=21569568768 total_nvlink_bytes=33281198768 weight_to_token_ratio=0.421290 prefetch_share_of_ingress=0.236846 dedup_ratio=0.200906 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=27 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020478 gemm_inner_pad_upper_bound=23368 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.548786345e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118932 payload_rows_remote=102548 dup_rows=12140 dup_rows_remote=570 dedup_ratio=0.092621 router_tokens_per_expert=[51, 0, 7314, 0, 8093, 144, 1, 8043, 8147, 3, 0, 0, 0, 0, 32, 1, 8153, 8, 5, 61, 13, 310, 0, 70, 8099, 0, 0, 23, 0, 5007, 8132, 3826] payload_rows_per_dst=[14730, 16384, 16300, 16268, 16336, 6550, 16242, 16122] dup_rows_per_dst=[0, 11570, 0, 62, 118, 148, 162, 80] combine_rows_per_src=[14730, 16384, 16300, 16268, 16336, 6550, 16242, 16122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129538 payload_rows_remote=113154 dup_rows=1534 dup_rows_remote=1488 dedup_ratio=0.011703 router_tokens_per_expert=[348, 135, 8192, 0, 7903, 2, 0, 7600, 8191, 0, 0, 0, 0, 0, 0, 23, 8168, 0, 0, 5, 815, 0, 0, 0, 8019, 0, 0, 0, 8, 3839, 8192, 4096] payload_rows_per_dst=[16384, 15806, 16382, 16384, 16336, 16384, 16038, 15824] dup_rows_per_dst=[966, 4, 0, 46, 10, 446, 0, 62] combine_rows_per_src=[16384, 15806, 16382, 16384, 16336, 16384, 16038, 15824] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=792100864 dispatch_meta_bytes=457424 combine_payload_bytes=792100864 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=792558288 ingress_bytes=1496743936 link_bytes=1496743936 token_bytes=1584659152 weight_movement_bytes=704643072 total_nvlink_bytes=2289302224 weight_to_token_ratio=0.444665 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.381141270e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=736821248 dispatch_meta_bytes=432376 combine_payload_bytes=736821248 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=737253624 ingress_bytes=1441464320 link_bytes=1441464320 token_bytes=1474074872 weight_movement_bytes=704643072 total_nvlink_bytes=2178717944 weight_to_token_ratio=0.478024 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=22144 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.071598495e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=846512128 dispatch_meta_bytes=459248 combine_payload_bytes=846512128 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=846971376 ingress_bytes=1198833664 link_bytes=1198833664 token_bytes=1693483504 weight_movement_bytes=352321536 total_nvlink_bytes=2045805040 weight_to_token_ratio=0.208045 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.712968101e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=109786 payload_rows_remote=93416 dup_rows=21286 dup_rows_remote=13712 dedup_ratio=0.162399 router_tokens_per_expert=[28, 3437, 8177, 0, 6067, 1, 7, 6275, 6745, 26, 0, 3, 804, 0, 120, 0, 8191, 0, 0, 50, 2822, 100, 0, 8, 7547, 0, 324, 2, 14, 445, 8068, 6275] payload_rows_per_dst=[11556, 9166, 13548, 14142, 16316, 12942, 15746, 16370] dup_rows_per_dst=[72, 14, 3488, 1600, 204, 5468, 2866, 7574] combine_rows_per_src=[11556, 9166, 13548, 14142, 16316, 12942, 15746, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126552 payload_rows_remote=110168 dup_rows=4520 dup_rows_remote=4520 dedup_ratio=0.034485 router_tokens_per_expert=[0, 1434, 7925, 10, 6558, 1, 0, 8180, 8188, 0, 0, 4, 0, 0, 0, 0, 8178, 0, 0, 12, 371, 0, 0, 23, 8114, 0, 76, 0, 216, 198, 8190, 7858] payload_rows_per_dst=[15870, 13130, 16384, 16380, 16376, 16304, 16380, 15728] dup_rows_per_dst=[2868, 90, 0, 0, 4, 742, 0, 816] combine_rows_per_src=[15870, 13130, 16384, 16380, 16376, 16304, 16380, 15728] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=840073216 dispatch_meta_bytes=412472 combine_payload_bytes=840073216 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=840485688 ingress_bytes=1897037824 link_bytes=1897037824 token_bytes=1680558904 weight_movement_bytes=1056964608 total_nvlink_bytes=2737523512 weight_to_token_ratio=0.628936 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=23296 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.062261995e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=926957568 dispatch_meta_bytes=458568 combine_payload_bytes=926957568 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=927416136 ingress_bytes=1279279104 link_bytes=1279279104 token_bytes=1854373704 weight_movement_bytes=352321536 total_nvlink_bytes=2206695240 weight_to_token_ratio=0.189995 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.163428985e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=765263872 dispatch_meta_bytes=428512 combine_payload_bytes=765263872 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=765692384 ingress_bytes=1998389248 link_bytes=1998389248 token_bytes=1530956256 weight_movement_bytes=1233125376 total_nvlink_bytes=2764081632 weight_to_token_ratio=0.805461 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=23296 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.119014562e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=24919701488 prefetch_bytes=5108662272 reduce_bytes=3875536896 weight_movement_bytes=8984199168 egress_bytes=12463388656 ingress_bytes=21440512000 link_bytes=21440512000 total_nvlink_bytes=33903900656 weight_to_token_ratio=0.360526 prefetch_share_of_ingress=0.238271 dedup_ratio=0.156075 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=22 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020241 gemm_inner_pad_upper_bound=23241 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.503619843e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=902496256 dispatch_meta_bytes=458752 combine_payload_bytes=902496256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=902955008 ingress_bytes=1254817792 link_bytes=1254817792 token_bytes=1805451264 weight_movement_bytes=352321536 total_nvlink_bytes=2157772800 weight_to_token_ratio=0.195143 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.026455848e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=25957432208 prefetch_bytes=5989466112 reduce_bytes=5284823040 weight_movement_bytes=11274289152 egress_bytes=12982320016 ingress_bytes=24249401344 link_bytes=24249401344 total_nvlink_bytes=37231721360 weight_to_token_ratio=0.434338 prefetch_share_of_ingress=0.246994 dedup_ratio=0.126684 prefetch_slots=68 unused_prefetch_slots=188 slot_utilization=0.265625 reduce_slots=30 remote_demand=34 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019650 gemm_inner_pad_upper_bound=24384 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.486657833e-05 prefetch_bytes_per_gemm_flop=2.096156881e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=25022676400 prefetch_bytes=5108662272 reduce_bytes=5284823040 weight_movement_bytes=10393485312 egress_bytes=12514901424 ingress_bytes=22901260288 link_bytes=22901260288 total_nvlink_bytes=35416161712 weight_to_token_ratio=0.415363 prefetch_share_of_ingress=0.223073 dedup_ratio=0.155649 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=30 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=23368 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.014843635e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111330 payload_rows_remote=98836 dup_rows=19742 dup_rows_remote=15702 dedup_ratio=0.150620 router_tokens_per_expert=[6812, 29, 4767, 0, 0, 3, 8021, 8100, 2, 0, 54, 1315, 147, 4570, 2, 45, 6959, 7, 7323, 151, 41, 11, 0, 0, 6577, 0, 0, 0, 0, 2472, 11, 8117] payload_rows_per_dst=[14682, 16208, 12296, 10190, 12494, 16046, 13158, 16256] dup_rows_per_dst=[1248, 6124, 2590, 498, 4040, 100, 198, 4944] combine_rows_per_src=[14682, 16208, 12296, 10190, 12494, 16046, 13158, 16256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118894 payload_rows_remote=102592 dup_rows=12178 dup_rows_remote=3974 dedup_ratio=0.092911 router_tokens_per_expert=[71, 1589, 8138, 0, 5136, 86, 111, 8079, 8158, 0, 0, 19, 0, 7, 8, 100, 8048, 30, 0, 0, 4041, 24, 80, 29, 7240, 0, 35, 17, 0, 122, 8163, 6205] payload_rows_per_dst=[16294, 10600, 16316, 16352, 16106, 16302, 14514, 12410] dup_rows_per_dst=[3302, 66, 38, 204, 50, 8204, 70, 244] combine_rows_per_src=[16294, 10600, 16316, 16352, 16106, 16302, 14514, 12410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=22600203592 prefetch_bytes=4227858432 reduce_bytes=9512681472 weight_movement_bytes=13740539904 egress_bytes=11303501128 ingress_bytes=25037242368 link_bytes=25037242368 total_nvlink_bytes=36340743496 weight_to_token_ratio=0.607983 prefetch_share_of_ingress=0.168863 dedup_ratio=0.221698 prefetch_slots=48 unused_prefetch_slots=208 slot_utilization=0.187500 reduce_slots=54 remote_demand=24 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020123 gemm_inner_pad_upper_bound=22733 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.762381638e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=25948031528 prefetch_bytes=5637144576 reduce_bytes=1937768448 weight_movement_bytes=7574913024 egress_bytes=12977621544 ingress_bytes=20545323008 link_bytes=20545323008 total_nvlink_bytes=33522944552 weight_to_token_ratio=0.291926 prefetch_share_of_ingress=0.274376 dedup_ratio=0.125118 prefetch_slots=64 unused_prefetch_slots=192 slot_utilization=0.250000 reduce_slots=11 remote_demand=32 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020123 gemm_inner_pad_upper_bound=24130 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.190327050e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=840433664 dispatch_meta_bytes=426264 combine_payload_bytes=840433664 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=840859928 ingress_bytes=1192755200 link_bytes=1192755200 token_bytes=1681293592 weight_movement_bytes=352321536 total_nvlink_bytes=2033615128 weight_to_token_ratio=0.209554 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.678931239e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=809664512 dispatch_meta_bytes=458152 combine_payload_bytes=809664512 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=810122664 ingress_bytes=1690468352 link_bytes=1690468352 token_bytes=1619787176 weight_movement_bytes=880803840 total_nvlink_bytes=2500591016 weight_to_token_ratio=0.543778 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=27904 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.465917135e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=26320432144 prefetch_bytes=5460983808 reduce_bytes=5108662272 weight_movement_bytes=10569646080 egress_bytes=13163883536 ingress_bytes=23726194688 link_bytes=23726194688 total_nvlink_bytes=36890078224 weight_to_token_ratio=0.401576 prefetch_share_of_ingress=0.230167 dedup_ratio=0.132511 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=29 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020360 gemm_inner_pad_upper_bound=23876 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.303549153e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107288 payload_rows_remote=91712 dup_rows=23784 dup_rows_remote=16386 dedup_ratio=0.181458 router_tokens_per_expert=[8177, 29, 0, 0, 3923, 39, 184, 7577, 2423, 0, 4082, 47, 3, 62, 6, 120, 6, 57, 0, 4342, 3924, 5714, 1, 367, 0, 8140, 4406, 0, 3846, 9, 0, 8052] payload_rows_per_dst=[16380, 8134, 12828, 9498, 15576, 12360, 16280, 16232] dup_rows_per_dst=[32, 158, 276, 368, 7398, 7652, 318, 7582] combine_rows_per_src=[16380, 8134, 12828, 9498, 15576, 12360, 16280, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=751304704 dispatch_meta_bytes=432392 combine_payload_bytes=751304704 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=751737096 ingress_bytes=1103626240 link_bytes=1103626240 token_bytes=1503041800 weight_movement_bytes=352321536 total_nvlink_bytes=1855363336 weight_to_token_ratio=0.234406 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=19200 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.179846268e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120748 payload_rows_remote=105264 dup_rows=10324 dup_rows_remote=10322 dedup_ratio=0.078766 router_tokens_per_expert=[15, 87, 23, 0, 7, 0, 80, 67, 13, 3880, 122, 8051, 4094, 1, 26, 0, 8126, 7741, 8171, 2, 7904, 20, 44, 8170, 4, 0, 5, 717, 8107, 3, 51, 5] payload_rows_per_dst=[16314, 16364, 7786, 16382, 15484, 15984, 16120, 16314] dup_rows_per_dst=[202, 272, 244, 8088, 2, 64, 1434, 18] combine_rows_per_src=[16314, 16364, 7786, 16382, 15484, 15984, 16120, 16314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=862322688 dispatch_meta_bytes=462344 combine_payload_bytes=862322688 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=862785032 ingress_bytes=1919287296 link_bytes=1919287296 token_bytes=1725107720 weight_movement_bytes=1056964608 total_nvlink_bytes=2782072328 weight_to_token_ratio=0.612695 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.074720771e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115786 payload_rows_remote=99450 dup_rows=15286 dup_rows_remote=10684 dedup_ratio=0.116623 router_tokens_per_expert=[5, 8180, 4837, 8186, 3949, 1, 8112, 1066, 6, 1, 9, 0, 21, 2, 14, 7, 415, 1241, 0, 713, 0, 8167, 0, 5, 0, 7596, 2587, 17, 2582, 7708, 98, 11] payload_rows_per_dst=[9684, 16012, 16346, 16384, 16336, 8324, 16384, 16316] dup_rows_per_dst=[0, 2102, 20, 64, 4602, 0, 4016, 4482] combine_rows_per_src=[9684, 16012, 16346, 16384, 16336, 8324, 16384, 16316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=814694400 dispatch_meta_bytes=440536 combine_payload_bytes=814694400 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=815134936 ingress_bytes=1167015936 link_bytes=1167015936 token_bytes=1629829336 weight_movement_bytes=352321536 total_nvlink_bytes=1982150872 weight_to_token_ratio=0.216171 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=23296 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.534802105e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117520 payload_rows_remote=101148 dup_rows=13552 dup_rows_remote=13498 dedup_ratio=0.103394 router_tokens_per_expert=[1, 15, 8088, 56, 5287, 0, 0, 7849, 8188, 71, 5, 2, 6, 0, 10, 171, 8186, 0, 0, 27, 5543, 38, 1, 7, 8138, 16, 4, 0, 194, 1404, 8192, 4037] payload_rows_per_dst=[16252, 10574, 16384, 16384, 16372, 16330, 16316, 8908] dup_rows_per_dst=[68, 0, 148, 374, 54, 10546, 0, 2362] combine_rows_per_src=[16252, 10574, 16384, 16384, 16372, 16330, 16316, 8908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=828604416 dispatch_meta_bytes=458584 combine_payload_bytes=828604416 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=829063000 ingress_bytes=1180925952 link_bytes=1180925952 token_bytes=1657667416 weight_movement_bytes=352321536 total_nvlink_bytes=2009988952 weight_to_token_ratio=0.212541 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.612692388e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=25662058656 prefetch_bytes=4756340736 reduce_bytes=5637144576 weight_movement_bytes=10393485312 egress_bytes=12834680992 ingress_bytes=23220862976 link_bytes=23220862976 total_nvlink_bytes=36055543968 weight_to_token_ratio=0.405014 prefetch_share_of_ingress=0.204830 dedup_ratio=0.140945 prefetch_slots=54 unused_prefetch_slots=202 slot_utilization=0.210938 reduce_slots=32 remote_demand=27 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=23241 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.126696238e-05 prefetch_bytes_per_gemm_flop=1.664595170e-05 anomalies=none - [2026-09-16 21:37:04] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 574.7 | throughput per GPU (TFLOP/s/GPU): 580.1 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.028801E+01 | loss scale: 1.0 | grad norm: 201.471 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98700 payload_rows_remote=83614 dup_rows=32372 dup_rows_remote=27440 dedup_ratio=0.246979 router_tokens_per_expert=[44, 6502, 3383, 158, 5733, 30, 1741, 97, 3074, 135, 6587, 1754, 1320, 845, 161, 590, 981, 2359, 3808, 7093, 5925, 1318, 2591, 414, 4, 3155, 379, 1280, 980, 170, 2468, 457] payload_rows_per_dst=[14138, 12628, 13644, 15086, 11368, 13940, 8684, 9212] dup_rows_per_dst=[6036, 2574, 5608, 4932, 2928, 6556, 952, 2786] combine_rows_per_src=[14138, 12628, 13644, 15086, 11368, 13940, 8684, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=93812 payload_rows_remote=78672 dup_rows=37260 dup_rows_remote=24952 dedup_ratio=0.284271 router_tokens_per_expert=[39, 6620, 3945, 103, 4824, 39, 1825, 108, 2594, 285, 6512, 1475, 867, 986, 253, 502, 864, 2516, 3693, 7142, 6091, 834, 2184, 416, 5, 2705, 394, 1670, 1582, 288, 2836, 1339] payload_rows_per_dst=[15140, 11366, 15478, 15144, 8758, 9896, 8342, 9688] dup_rows_per_dst=[12308, 2226, 10046, 4356, 1596, 3120, 1206, 2402] combine_rows_per_src=[15140, 11366, 15478, 15144, 8758, 9896, 8342, 9688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=92608 payload_rows_remote=77542 dup_rows=38464 dup_rows_remote=31012 dedup_ratio=0.293457 router_tokens_per_expert=[43, 6352, 3039, 92, 5745, 12, 2180, 105, 3182, 170, 6563, 1651, 738, 1092, 133, 487, 783, 2989, 3231, 7269, 6416, 885, 2722, 304, 8, 3392, 383, 1274, 727, 151, 2574, 844] payload_rows_per_dst=[6294, 13212, 10830, 5904, 15254, 11178, 15066, 14870] dup_rows_per_dst=[54, 4704, 3340, 784, 12316, 2966, 7452, 6848] combine_rows_per_src=[6294, 13212, 10830, 5904, 15254, 11178, 15066, 14870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=99180 payload_rows_remote=83980 dup_rows=31892 dup_rows_remote=25422 dedup_ratio=0.243317 router_tokens_per_expert=[59, 6976, 3443, 40, 5183, 37, 1647, 128, 2960, 119, 6400, 1356, 929, 1258, 78, 572, 881, 2281, 3589, 7271, 6047, 554, 2728, 375, 1, 4042, 437, 1593, 1076, 267, 2424, 785] payload_rows_per_dst=[14842, 12166, 15200, 15326, 11030, 12638, 10234, 7744] dup_rows_per_dst=[6194, 3612, 6470, 4890, 2472, 4982, 1912, 1360] combine_rows_per_src=[14842, 12166, 15200, 15326, 11030, 12638, 10234, 7744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97772 payload_rows_remote=86736 dup_rows=33300 dup_rows_remote=29690 dedup_ratio=0.254059 router_tokens_per_expert=[74, 6419, 3216, 59, 5505, 17, 2210, 86, 3683, 176, 6924, 1673, 878, 1292, 261, 515, 931, 1722, 3130, 6861, 6306, 1676, 1964, 544, 2, 3077, 457, 1273, 1038, 213, 2621, 733] payload_rows_per_dst=[8510, 13322, 9870, 14956, 9786, 11036, 14992, 15300] dup_rows_per_dst=[1562, 5084, 1194, 4658, 1780, 3610, 7654, 7758] combine_rows_per_src=[8510, 13322, 9870, 14956, 9786, 11036, 14992, 15300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=91574 payload_rows_remote=76756 dup_rows=39498 dup_rows_remote=31614 dedup_ratio=0.301346 router_tokens_per_expert=[58, 6442, 3360, 39, 5241, 32, 1893, 66, 2144, 231, 6436, 1845, 857, 1094, 195, 541, 954, 2429, 3194, 7567, 6229, 1244, 2626, 373, 2, 3485, 410, 1634, 1240, 336, 2541, 798] payload_rows_per_dst=[6848, 11960, 7626, 4904, 15984, 14230, 15204, 14818] dup_rows_per_dst=[66, 2504, 814, 470, 11706, 7146, 8908, 7884] combine_rows_per_src=[6848, 11960, 7626, 4904, 15984, 14230, 15204, 14818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95086 payload_rows_remote=81312 dup_rows=35986 dup_rows_remote=30544 dedup_ratio=0.274551 router_tokens_per_expert=[32, 6791, 3871, 114, 5928, 18, 1699, 112, 2274, 194, 6143, 1621, 897, 907, 204, 492, 872, 2710, 3251, 7214, 6345, 1183, 2277, 299, 15, 3165, 573, 1569, 1176, 444, 2155, 991] payload_rows_per_dst=[14680, 13774, 15038, 15222, 8940, 10522, 9048, 7862] dup_rows_per_dst=[9862, 5442, 7982, 4206, 2170, 3058, 1596, 1670] combine_rows_per_src=[14680, 13774, 15038, 15222, 8940, 10522, 9048, 7862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=644481024 dispatch_meta_bytes=414496 combine_payload_bytes=644481024 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=644895520 ingress_bytes=1701445632 link_bytes=1701445632 token_bytes=1289376544 weight_movement_bytes=1056964608 total_nvlink_bytes=2346341152 weight_to_token_ratio=0.819749 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=18560 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.527385321e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=684965888 dispatch_meta_bytes=444216 combine_payload_bytes=684965888 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=685410104 ingress_bytes=1037287424 link_bytes=1037287424 token_bytes=1370375992 weight_movement_bytes=352321536 total_nvlink_bytes=1722697528 weight_to_token_ratio=0.257098 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=22144 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.808376589e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=687964160 dispatch_meta_bytes=437608 combine_payload_bytes=687964160 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=688401768 ingress_bytes=1392607232 link_bytes=1392607232 token_bytes=1376365928 weight_movement_bytes=704643072 total_nvlink_bytes=2081009000 weight_to_token_ratio=0.511959 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=12160 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.798019196e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=635224064 dispatch_meta_bytes=434216 combine_payload_bytes=635224064 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=635658280 ingress_bytes=987545600 link_bytes=987545600 token_bytes=1270882344 weight_movement_bytes=352321536 total_nvlink_bytes=1623203880 weight_to_token_ratio=0.277226 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=13568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.529843138e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=99450 payload_rows_remote=87934 dup_rows=31622 dup_rows_remote=28154 dedup_ratio=0.241257 router_tokens_per_expert=[27, 6383, 3654, 133, 6125, 18, 2040, 66, 2739, 127, 5833, 1100, 1040, 790, 195, 642, 892, 3404, 3196, 7450, 6148, 802, 2953, 508, 2, 2941, 425, 1301, 1132, 119, 2452, 899] payload_rows_per_dst=[11314, 13290, 7398, 15480, 11516, 12010, 14130, 14312] dup_rows_per_dst=[3224, 3208, 534, 4754, 3468, 4184, 5692, 6558] combine_rows_per_src=[11314, 13290, 7398, 15480, 11516, 12010, 14130, 14312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=710541312 dispatch_meta_bytes=465704 combine_payload_bytes=710541312 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=711007016 ingress_bytes=1591345152 link_bytes=1591345152 token_bytes=1421548328 weight_movement_bytes=880803840 total_nvlink_bytes=2302352168 weight_to_token_ratio=0.619609 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=17152 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.910868591e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=628785152 dispatch_meta_bytes=433480 combine_payload_bytes=628785152 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=629218632 ingress_bytes=981106688 link_bytes=981106688 token_bytes=1258003784 weight_movement_bytes=352321536 total_nvlink_bytes=1610325320 weight_to_token_ratio=0.280064 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=14336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.493787919e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=666107904 dispatch_meta_bytes=447424 combine_payload_bytes=666107904 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=666555328 ingress_bytes=1018429440 link_bytes=1018429440 token_bytes=1332663232 weight_movement_bytes=352321536 total_nvlink_bytes=1684984768 weight_to_token_ratio=0.264374 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=22272 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.702779751e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=720355328 dispatch_meta_bytes=464352 combine_payload_bytes=720355328 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=720819680 ingress_bytes=1601159168 link_bytes=1601159168 token_bytes=1441175008 weight_movement_bytes=880803840 total_nvlink_bytes=2321978848 weight_to_token_ratio=0.611171 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=12416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.965822984e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122300 payload_rows_remote=105934 dup_rows=8772 dup_rows_remote=2764 dedup_ratio=0.066925 router_tokens_per_expert=[0, 8144, 2943, 100, 4, 4, 22, 2, 7, 7859, 14, 0, 9, 9, 7589, 0, 12, 7, 4, 1, 7999, 0, 59, 130, 8109, 5437, 8183, 22, 1102, 2, 31, 7732] payload_rows_per_dst=[16366, 16246, 15726, 15194, 16382, 16056, 10620, 15710] dup_rows_per_dst=[6008, 334, 34, 20, 32, 320, 0, 2024] combine_rows_per_src=[16366, 16246, 15726, 15194, 16382, 16056, 10620, 15710] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126160 payload_rows_remote=109968 dup_rows=4912 dup_rows_remote=4906 dedup_ratio=0.037476 router_tokens_per_expert=[1, 8163, 1662, 359, 12, 1, 2, 224, 11, 7899, 1, 2, 1, 15, 8082, 1, 21, 1, 1, 1, 7963, 39, 19, 219, 8074, 6460, 8181, 2, 173, 0, 8, 7938] payload_rows_per_dst=[16338, 16224, 15820, 16192, 16376, 16272, 12924, 16014] dup_rows_per_dst=[4032, 402, 6, 6, 34, 208, 0, 224] combine_rows_per_src=[16338, 16224, 15820, 16192, 16376, 16272, 12924, 16014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114250 payload_rows_remote=98710 dup_rows=16822 dup_rows_remote=6990 dedup_ratio=0.128342 router_tokens_per_expert=[13, 8189, 1697, 459, 0, 0, 192, 35, 0, 7912, 15, 0, 0, 0, 8063, 1, 10, 8, 4, 0, 8124, 1, 3, 85, 8074, 7714, 8190, 2, 12, 0, 97, 6636] payload_rows_per_dst=[4078, 16298, 15848, 16360, 16384, 16310, 15540, 13432] dup_rows_per_dst=[260, 304, 6, 4464, 40, 1858, 9832, 58] combine_rows_per_src=[4078, 16298, 15848, 16360, 16384, 16310, 15540, 13432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123622 payload_rows_remote=107278 dup_rows=7450 dup_rows_remote=7332 dedup_ratio=0.056839 router_tokens_per_expert=[10, 8190, 2977, 70, 23, 0, 17, 78, 0, 8168, 61, 2, 0, 0, 7871, 4, 0, 25, 0, 0, 7971, 2, 1, 65, 8111, 5544, 8171, 8, 374, 0, 622, 7171] payload_rows_per_dst=[16384, 16302, 16344, 15748, 16372, 15990, 11096, 15386] dup_rows_per_dst=[6110, 156, 118, 2, 20, 88, 8, 948] combine_rows_per_src=[16384, 16302, 16344, 15748, 16372, 15990, 11096, 15386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 2, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 3, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122348 payload_rows_remote=106206 dup_rows=8724 dup_rows_remote=8656 dedup_ratio=0.066559 router_tokens_per_expert=[0, 8190, 1874, 300, 4, 0, 3, 221, 2, 7874, 0, 5, 0, 0, 8013, 2, 0, 0, 5, 0, 8062, 1, 30, 12, 8158, 7733, 8185, 0, 213, 1, 48, 6600] payload_rows_per_dst=[12652, 16332, 15754, 16046, 16374, 16142, 15466, 13582] dup_rows_per_dst=[3272, 440, 2666, 2130, 6, 68, 0, 142] combine_rows_per_src=[12652, 16332, 15754, 16046, 16374, 16142, 15466, 13582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113008 payload_rows_remote=97622 dup_rows=18064 dup_rows_remote=9904 dedup_ratio=0.137817 router_tokens_per_expert=[7, 8165, 1385, 503, 0, 1, 134, 11, 0, 8116, 5, 2, 1, 81, 7029, 12, 1, 19, 4, 6, 8097, 11, 2, 116, 7889, 8154, 8159, 5, 347, 1, 178, 7095] payload_rows_per_dst=[3398, 14908, 16240, 14194, 16326, 16214, 16342, 15386] dup_rows_per_dst=[392, 106, 6, 52, 16, 238, 9094, 8160] combine_rows_per_src=[3398, 14908, 16240, 14194, 16326, 16214, 16342, 15386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=867811328 dispatch_meta_bytes=434792 combine_payload_bytes=867811328 prefetch_bytes=0 reduce_bytes=1761607680 egress_bytes=868246120 ingress_bytes=2629419008 link_bytes=2629419008 token_bytes=1736057448 weight_movement_bytes=1761607680 total_nvlink_bytes=3497665128 weight_to_token_ratio=1.014717 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.472364887e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122510 payload_rows_remote=106208 dup_rows=8562 dup_rows_remote=8444 dedup_ratio=0.065323 router_tokens_per_expert=[57, 8152, 2598, 1102, 0, 1, 34, 72, 0, 7988, 88, 1, 1, 0, 7536, 1, 0, 0, 0, 0, 7185, 46, 164, 5, 8103, 7222, 8179, 10, 1411, 18, 247, 5315] payload_rows_per_dst=[16364, 16302, 16140, 15076, 16358, 14544, 14456, 13270] dup_rows_per_dst=[7454, 118, 14, 0, 0, 256, 8, 712] combine_rows_per_src=[16364, 16302, 16140, 15076, 16358, 14544, 14456, 13270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=900857856 dispatch_meta_bytes=459496 combine_payload_bytes=900857856 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=901317352 ingress_bytes=1253179392 link_bytes=1253179392 token_bytes=1802175208 weight_movement_bytes=352321536 total_nvlink_bytes=2154496744 weight_to_token_ratio=0.195498 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.017281492e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=808632320 dispatch_meta_bytes=422800 combine_payload_bytes=808632320 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=809055120 ingress_bytes=2041757696 link_bytes=2041757696 token_bytes=1617687440 weight_movement_bytes=1233125376 total_nvlink_bytes=2850812816 weight_to_token_ratio=0.762277 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=28032 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.143299083e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=878821376 dispatch_meta_bytes=458440 combine_payload_bytes=878821376 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=879279816 ingress_bytes=1231142912 link_bytes=1231142912 token_bytes=1758101192 weight_movement_bytes=352321536 total_nvlink_bytes=2110422728 weight_to_token_ratio=0.200399 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.893886403e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 6, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=799719424 dispatch_meta_bytes=430104 combine_payload_bytes=799719424 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=800149528 ingress_bytes=1152040960 link_bytes=1152040960 token_bytes=1599868952 weight_movement_bytes=352321536 total_nvlink_bytes=1952190488 weight_to_token_ratio=0.220219 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=29056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.450948490e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=870039552 dispatch_meta_bytes=459448 combine_payload_bytes=870039552 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=870499000 ingress_bytes=1222361088 link_bytes=1222361088 token_bytes=1740538552 weight_movement_bytes=352321536 total_nvlink_bytes=2092860088 weight_to_token_ratio=0.202421 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.844711854e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=870055936 dispatch_meta_bytes=458608 combine_payload_bytes=870055936 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=870514544 ingress_bytes=1398538240 link_bytes=1398538240 token_bytes=1740570480 weight_movement_bytes=528482304 total_nvlink_bytes=2269052784 weight_to_token_ratio=0.303626 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.017045 gemm_active_groups=13 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.831230365e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128862 payload_rows_remote=112510 dup_rows=2210 dup_rows_remote=2210 dedup_ratio=0.016861 router_tokens_per_expert=[56, 8171, 700, 55, 67, 0, 22, 37, 11, 8152, 8, 4, 0, 0, 7787, 7, 1, 0, 0, 0, 8158, 10, 49, 37, 8174, 7843, 8175, 0, 250, 0, 11, 7751] payload_rows_per_dst=[16384, 16384, 16318, 15588, 16352, 16372, 15686, 15778] dup_rows_per_dst=[1580, 216, 32, 0, 0, 136, 0, 246] combine_rows_per_src=[16384, 16384, 16318, 15588, 16352, 16372, 15686, 15778] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118260 payload_rows_remote=101884 dup_rows=12812 dup_rows_remote=12712 dedup_ratio=0.097748 router_tokens_per_expert=[72, 5, 2, 5, 76, 8019, 25, 8161, 0, 7832, 90, 8, 8157, 8140, 8081, 136, 8171, 23, 16, 86, 72, 38, 11, 3, 80, 7698, 479, 3, 8, 7, 32, 0] payload_rows_per_dst=[16376, 16384, 16342, 14904, 9880, 13894, 16112, 14368] dup_rows_per_dst=[100, 6486, 4780, 898, 32, 60, 408, 48] combine_rows_per_src=[16376, 16384, 16342, 14904, 9880, 13894, 16112, 14368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125954 payload_rows_remote=110040 dup_rows=5118 dup_rows_remote=2532 dedup_ratio=0.039047 router_tokens_per_expert=[7, 0, 7, 0, 2, 7973, 0, 8192, 3, 5883, 7, 1626, 8187, 8191, 8191, 141, 8185, 0, 0, 399, 0, 177, 1, 2, 0, 7916, 33, 2, 27, 0, 384, 0] payload_rows_per_dst=[16384, 15950, 12184, 16384, 16370, 16384, 15914, 16384] dup_rows_per_dst=[18, 0, 42, 280, 1012, 358, 2586, 822] combine_rows_per_src=[16384, 15950, 12184, 16384, 16370, 16384, 15914, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127010 payload_rows_remote=110740 dup_rows=4062 dup_rows_remote=4022 dedup_ratio=0.030991 router_tokens_per_expert=[2, 0, 25, 0, 0, 8168, 3, 8184, 21, 7499, 1, 1425, 8172, 8185, 8152, 21, 8178, 0, 0, 5, 0, 430, 0, 1, 27, 6652, 55, 0, 1, 0, 329, 0] payload_rows_per_dst=[16380, 16336, 15510, 16270, 16362, 16374, 13394, 16384] dup_rows_per_dst=[48, 6, 2382, 40, 4, 852, 74, 656] combine_rows_per_src=[16380, 16336, 15510, 16270, 16362, 16374, 13394, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124300 payload_rows_remote=110600 dup_rows=6772 dup_rows_remote=3764 dedup_ratio=0.051666 router_tokens_per_expert=[3, 0, 17, 0, 6, 7792, 0, 8153, 0, 6532, 0, 1822, 8191, 8185, 8183, 56, 8185, 1, 0, 1697, 0, 72, 0, 5, 53, 5938, 613, 0, 26, 0, 0, 6] payload_rows_per_dst=[16374, 15590, 13700, 16378, 16382, 16380, 13148, 16348] dup_rows_per_dst=[40, 6, 3008, 102, 3384, 154, 60, 18] combine_rows_per_src=[16374, 15590, 13700, 16378, 16382, 16380, 13148, 16348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=834633728 dispatch_meta_bytes=458384 combine_payload_bytes=834633728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=835092112 ingress_bytes=1186955264 link_bytes=1186955264 token_bytes=1669725840 weight_movement_bytes=352321536 total_nvlink_bytes=2022047376 weight_to_token_ratio=0.211006 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.646454019e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=921681920 dispatch_meta_bytes=458880 combine_payload_bytes=921681920 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=922140800 ingress_bytes=1274003456 link_bytes=1274003456 token_bytes=1843822720 weight_movement_bytes=352321536 total_nvlink_bytes=2196144256 weight_to_token_ratio=0.191082 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.133887558e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121330 payload_rows_remote=104948 dup_rows=9742 dup_rows_remote=9382 dedup_ratio=0.074326 router_tokens_per_expert=[8, 0, 21, 6, 0, 6384, 0, 8172, 87, 7415, 1, 2390, 8181, 8150, 8185, 0, 8139, 0, 9, 33, 0, 1, 49, 37, 18, 7867, 175, 6, 164, 1, 36, 1] payload_rows_per_dst=[16012, 11540, 12896, 15536, 16304, 16384, 16276, 16382] dup_rows_per_dst=[48, 0, 50, 422, 1032, 96, 7734, 360] combine_rows_per_src=[16012, 11540, 12896, 15536, 16304, 16384, 16276, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=901447680 dispatch_meta_bytes=450288 combine_payload_bytes=901447680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=901897968 ingress_bytes=1253769216 link_bytes=1253769216 token_bytes=1803345648 weight_movement_bytes=352321536 total_nvlink_bytes=2155667184 weight_to_token_ratio=0.195371 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.020584261e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126840 payload_rows_remote=110582 dup_rows=4232 dup_rows_remote=4226 dedup_ratio=0.032288 router_tokens_per_expert=[0, 35, 0, 2, 12, 8120, 0, 8184, 0, 6408, 0, 2790, 8131, 8192, 8157, 154, 8142, 0, 18, 451, 0, 119, 1, 2, 1, 6435, 89, 0, 82, 0, 11, 0] payload_rows_per_dst=[16384, 16258, 15784, 16272, 16376, 16382, 13044, 16340] dup_rows_per_dst=[68, 6, 2612, 280, 846, 238, 6, 176] combine_rows_per_src=[16384, 16258, 15784, 16272, 16376, 16382, 13044, 16340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129686 payload_rows_remote=113302 dup_rows=1386 dup_rows_remote=846 dedup_ratio=0.010574 router_tokens_per_expert=[1, 42, 0, 1, 20, 8157, 15, 8188, 2, 7796, 0, 30, 8180, 8190, 8167, 128, 8185, 4, 0, 154, 0, 267, 1, 3, 31, 7794, 178, 0, 0, 0, 2, 0] payload_rows_per_dst=[16378, 16352, 15652, 16354, 16378, 16384, 15848, 16340] dup_rows_per_dst=[84, 32, 4, 256, 308, 540, 158, 4] combine_rows_per_src=[16378, 16352, 15652, 16354, 16378, 16384, 15848, 16340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=907182080 dispatch_meta_bytes=459048 combine_payload_bytes=907182080 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=907641128 ingress_bytes=2316468224 link_bytes=2316468224 token_bytes=1814823208 weight_movement_bytes=1409286144 total_nvlink_bytes=3224109352 weight_to_token_ratio=0.776542 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.297125511e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=906035200 dispatch_meta_bytes=457456 combine_payload_bytes=906035200 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=906492656 ingress_bytes=1434517504 link_bytes=1434517504 token_bytes=1812527856 weight_movement_bytes=528482304 total_nvlink_bytes=2341010160 weight_to_token_ratio=0.291572 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=27264 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.032699225e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=859734016 dispatch_meta_bytes=457320 combine_payload_bytes=859734016 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=860191336 ingress_bytes=1212055552 link_bytes=1212055552 token_bytes=1719925352 weight_movement_bytes=352321536 total_nvlink_bytes=2072246888 weight_to_token_ratio=0.204847 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.787005154e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=905887744 dispatch_meta_bytes=459232 combine_payload_bytes=905887744 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=906346976 ingress_bytes=1786691584 link_bytes=1786691584 token_bytes=1812234720 weight_movement_bytes=880803840 total_nvlink_bytes=2693038560 weight_to_token_ratio=0.486032 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.000472707e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=928169984 dispatch_meta_bytes=456592 combine_payload_bytes=928169984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928626576 ingress_bytes=1280491520 link_bytes=1280491520 token_bytes=1856796560 weight_movement_bytes=352321536 total_nvlink_bytes=2209118096 weight_to_token_ratio=0.189747 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.170218009e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121586 payload_rows_remote=105244 dup_rows=9486 dup_rows_remote=8780 dedup_ratio=0.072372 router_tokens_per_expert=[30, 0, 32, 8133, 11, 3, 465, 1, 8130, 20, 185, 0, 8138, 7, 8156, 7973, 2, 0, 210, 9, 0, 8134, 4, 7567, 43, 0, 4, 106, 0, 8169, 4, 0] payload_rows_per_dst=[16342, 16376, 16382, 16368, 12234, 14512, 15088, 14284] dup_rows_per_dst=[706, 874, 280, 7574, 6, 0, 42, 4] combine_rows_per_src=[16342, 16376, 16382, 16368, 12234, 14512, 15088, 14284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129916 payload_rows_remote=113534 dup_rows=1156 dup_rows_remote=1154 dedup_ratio=0.008820 router_tokens_per_expert=[0, 2, 165, 8192, 0, 0, 0, 12, 8192, 0, 3, 0, 8191, 0, 8188, 7719, 4, 0, 300, 0, 0, 8192, 0, 8091, 0, 1, 0, 0, 0, 8179, 102, 3] payload_rows_per_dst=[16384, 16380, 16384, 15438, 16384, 16182, 16382, 16382] dup_rows_per_dst=[334, 20, 6, 0, 608, 0, 2, 186] combine_rows_per_src=[16384, 16380, 16384, 15438, 16384, 16182, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127470 payload_rows_remote=111098 dup_rows=3602 dup_rows_remote=3414 dedup_ratio=0.027481 router_tokens_per_expert=[97, 0, 644, 8149, 0, 0, 509, 0, 8186, 0, 94, 0, 8192, 1, 8192, 6933, 1, 15, 47, 0, 21, 8146, 19, 7642, 124, 0, 0, 26, 0, 8177, 315, 6] payload_rows_per_dst=[16376, 16384, 16372, 13868, 16368, 15360, 16384, 16358] dup_rows_per_dst=[1404, 1008, 188, 0, 60, 4, 300, 638] combine_rows_per_src=[16376, 16384, 16372, 13868, 16368, 15360, 16384, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130102 payload_rows_remote=113724 dup_rows=970 dup_rows_remote=946 dedup_ratio=0.007401 router_tokens_per_expert=[0, 0, 0, 4, 4, 8059, 0, 8190, 0, 6839, 6, 1218, 8185, 8184, 8186, 162, 8180, 0, 0, 21, 0, 109, 0, 3, 1, 8041, 111, 2, 13, 0, 18, 0] payload_rows_per_dst=[16376, 16126, 15920, 16374, 16378, 16384, 16160, 16384] dup_rows_per_dst=[2, 0, 206, 322, 24, 208, 150, 58] combine_rows_per_src=[16376, 16126, 15920, 16374, 16378, 16384, 16160, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130028 payload_rows_remote=114518 dup_rows=1044 dup_rows_remote=1036 dedup_ratio=0.007965 router_tokens_per_expert=[0, 0, 135, 8175, 2, 1, 280, 0, 8138, 10, 17, 0, 8184, 5, 8165, 7754, 15, 4, 33, 0, 1, 8186, 13, 8159, 0, 0, 2, 62, 5, 8185, 0, 5] payload_rows_per_dst=[16350, 16382, 16328, 15510, 16372, 16334, 16374, 16378] dup_rows_per_dst=[270, 558, 2, 8, 60, 12, 122, 12] combine_rows_per_src=[16350, 16382, 16328, 15510, 16372, 16334, 16374, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=862158848 dispatch_meta_bytes=456096 combine_payload_bytes=862158848 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=862614944 ingress_bytes=1566801920 link_bytes=1566801920 token_bytes=1724773792 weight_movement_bytes=704643072 total_nvlink_bytes=2429416864 weight_to_token_ratio=0.408542 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.773436737e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122212 payload_rows_remote=105828 dup_rows=8860 dup_rows_remote=8358 dedup_ratio=0.067596 router_tokens_per_expert=[21, 8, 18, 8178, 1, 0, 1, 5, 8148, 5, 7, 1, 8167, 16, 8184, 7385, 0, 0, 765, 16, 2, 8179, 7, 8011, 26, 4, 0, 8, 4, 8179, 179, 11] payload_rows_per_dst=[12788, 13708, 15922, 14802, 16216, 16030, 16362, 16384] dup_rows_per_dst=[68, 6, 6, 2832, 1550, 3848, 48, 502] combine_rows_per_src=[12788, 13708, 15922, 14802, 16216, 16030, 16362, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=930070528 dispatch_meta_bytes=458752 combine_payload_bytes=930070528 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930529280 ingress_bytes=1282392064 link_bytes=1282392064 token_bytes=1860599808 weight_movement_bytes=352321536 total_nvlink_bytes=2212921344 weight_to_token_ratio=0.189359 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.180860262e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=910114816 dispatch_meta_bytes=458048 combine_payload_bytes=910114816 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=910572864 ingress_bytes=1614757888 link_bytes=1614757888 token_bytes=1820687680 weight_movement_bytes=704643072 total_nvlink_bytes=2525330752 weight_to_token_ratio=0.387020 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.041970140e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129340 payload_rows_remote=112966 dup_rows=1732 dup_rows_remote=1732 dedup_ratio=0.013214 router_tokens_per_expert=[47, 0, 59, 8179, 0, 0, 105, 0, 8154, 12, 12, 1, 8191, 0, 8182, 7405, 0, 1, 22, 1, 0, 8162, 0, 8187, 36, 82, 5, 284, 7, 8182, 218, 2] payload_rows_per_dst=[16358, 16368, 16318, 14810, 16346, 16374, 16384, 16382] dup_rows_per_dst=[212, 210, 40, 0, 22, 0, 812, 436] combine_rows_per_src=[16358, 16368, 16318, 14810, 16346, 16374, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129996 payload_rows_remote=113612 dup_rows=1076 dup_rows_remote=1058 dedup_ratio=0.008209 router_tokens_per_expert=[303, 0, 65, 8172, 0, 0, 10, 0, 8180, 0, 47, 4, 8189, 8, 8192, 7872, 2, 1, 98, 0, 0, 8161, 7, 8006, 0, 2, 0, 1, 5, 8163, 48, 0] payload_rows_per_dst=[16370, 16384, 16368, 15744, 16384, 16024, 16382, 16340] dup_rows_per_dst=[710, 18, 94, 16, 142, 2, 2, 92] combine_rows_per_src=[16370, 16384, 16368, 15744, 16384, 16024, 16382, 16340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938131456 dispatch_meta_bytes=462216 combine_payload_bytes=938131456 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=938593672 ingress_bytes=1995096064 link_bytes=1995096064 token_bytes=1876725128 weight_movement_bytes=1056964608 total_nvlink_bytes=2933689736 weight_to_token_ratio=0.563196 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117170516e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=931627008 dispatch_meta_bytes=458680 combine_payload_bytes=931627008 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=932085688 ingress_bytes=1636270080 link_bytes=1636270080 token_bytes=1863712696 weight_movement_bytes=704643072 total_nvlink_bytes=2568355768 weight_to_token_ratio=0.378086 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.162429435e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=866942976 dispatch_meta_bytes=456744 combine_payload_bytes=866942976 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=867399720 ingress_bytes=1395425280 link_bytes=1395425280 token_bytes=1734342696 weight_movement_bytes=528482304 total_nvlink_bytes=2262825000 weight_to_token_ratio=0.304716 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.813799089e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=925417472 dispatch_meta_bytes=458792 combine_payload_bytes=925417472 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=925876264 ingress_bytes=1806221312 link_bytes=1806221312 token_bytes=1851293736 weight_movement_bytes=880803840 total_nvlink_bytes=2732097576 weight_to_token_ratio=0.475777 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.011408539e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=930709504 dispatch_meta_bytes=458680 combine_payload_bytes=930709504 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=931168184 ingress_bytes=1283031040 link_bytes=1283031040 token_bytes=1861877688 weight_movement_bytes=352321536 total_nvlink_bytes=2214199224 weight_to_token_ratio=0.189229 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.184438260e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120890 payload_rows_remote=104568 dup_rows=10182 dup_rows_remote=10174 dedup_ratio=0.077682 router_tokens_per_expert=[2, 3, 4, 0, 36, 0, 8161, 2231, 0, 8110, 0, 474, 0, 79, 8181, 15, 3546, 4533, 44, 14, 8159, 8, 15, 25, 8128, 1, 1, 7, 33, 8156, 186, 5384] payload_rows_per_dst=[16322, 16384, 12920, 16384, 15096, 16378, 16272, 11134] dup_rows_per_dst=[8, 7792, 930, 164, 1178, 36, 2, 72] combine_rows_per_src=[16322, 16384, 12920, 16384, 15096, 16378, 16272, 11134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118608 payload_rows_remote=102234 dup_rows=12464 dup_rows_remote=12464 dedup_ratio=0.095093 router_tokens_per_expert=[0, 0, 28, 24, 0, 0, 8183, 836, 0, 8116, 0, 655, 0, 8, 8147, 0, 3289, 5222, 0, 0, 8192, 0, 0, 0, 8187, 0, 0, 0, 0, 8192, 0, 6457] payload_rows_per_dst=[16384, 12162, 13074, 16296, 14716, 16384, 16374, 13218] dup_rows_per_dst=[104, 1558, 922, 14, 2306, 0, 0, 7560] combine_rows_per_src=[16384, 12162, 13074, 16296, 14716, 16384, 16374, 13218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128338 payload_rows_remote=111994 dup_rows=2734 dup_rows_remote=1970 dedup_ratio=0.020859 router_tokens_per_expert=[0, 0, 0, 194, 0, 0, 8175, 698, 17, 8136, 0, 401, 0, 79, 8192, 3, 2676, 4843, 7, 0, 8084, 3, 0, 0, 8192, 1, 0, 3, 0, 8177, 4, 7651] payload_rows_per_dst=[16384, 16354, 16344, 16384, 15004, 16174, 16384, 15310] dup_rows_per_dst=[358, 1392, 764, 164, 48, 0, 8, 0] combine_rows_per_src=[16384, 16354, 16344, 16384, 15004, 16174, 16384, 15310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=856621056 dispatch_meta_bytes=458968 combine_payload_bytes=856621056 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=857080024 ingress_bytes=1208942592 link_bytes=1208942592 token_bytes=1713701080 weight_movement_bytes=352321536 total_nvlink_bytes=2066022616 weight_to_token_ratio=0.205591 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.769573878e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128738 payload_rows_remote=112672 dup_rows=2334 dup_rows_remote=2320 dedup_ratio=0.017807 router_tokens_per_expert=[0, 1, 0, 24, 0, 0, 8186, 40, 23, 7246, 32, 1392, 3, 12, 8020, 5, 5341, 2795, 1, 0, 8190, 0, 0, 3, 8190, 1, 2, 5, 1, 8082, 53, 7888] payload_rows_per_dst=[16200, 16376, 16126, 16066, 15414, 16384, 16384, 15788] dup_rows_per_dst=[14, 76, 1260, 14, 860, 2, 12, 96] combine_rows_per_src=[16200, 16376, 16126, 16066, 15414, 16384, 16384, 15788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111704 payload_rows_remote=95412 dup_rows=19368 dup_rows_remote=10362 dedup_ratio=0.147766 router_tokens_per_expert=[0, 2, 15, 0, 13, 5, 7885, 4031, 0, 8170, 3, 953, 6, 50, 8177, 1, 4407, 3056, 0, 3, 8028, 0, 0, 5, 8145, 0, 3, 0, 3, 8175, 172, 4228] payload_rows_per_dst=[15832, 8070, 8846, 14748, 15558, 16064, 16294, 16292] dup_rows_per_dst=[28, 28, 1684, 86, 7730, 640, 166, 9006] combine_rows_per_src=[15832, 8070, 8846, 14748, 15558, 16064, 16294, 16292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128760 payload_rows_remote=112386 dup_rows=2312 dup_rows_remote=366 dedup_ratio=0.017639 router_tokens_per_expert=[0, 0, 15, 8185, 3, 1, 0, 2, 8191, 0, 0, 0, 8190, 0, 8192, 7051, 0, 2, 976, 0, 0, 8182, 3, 8192, 0, 0, 0, 4, 0, 8187, 159, 1] payload_rows_per_dst=[16370, 16384, 16382, 14102, 16374, 16384, 16380, 16384] dup_rows_per_dst=[30, 12, 0, 0, 1946, 6, 8, 310] combine_rows_per_src=[16370, 16384, 16382, 14102, 16374, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=837500928 dispatch_meta_bytes=458792 combine_payload_bytes=837500928 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=837959720 ingress_bytes=1189822464 link_bytes=1189822464 token_bytes=1675460648 weight_movement_bytes=352321536 total_nvlink_bytes=2027782184 weight_to_token_ratio=0.210283 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.662509142e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=917454848 dispatch_meta_bytes=455856 combine_payload_bytes=917454848 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=917910704 ingress_bytes=1798258688 link_bytes=1798258688 token_bytes=1835365552 weight_movement_bytes=880803840 total_nvlink_bytes=2716169392 weight_to_token_ratio=0.479906 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=29056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.006949802e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125550 payload_rows_remote=109318 dup_rows=5522 dup_rows_remote=5522 dedup_ratio=0.042130 router_tokens_per_expert=[0, 0, 23, 2, 0, 0, 8152, 161, 0, 8186, 4, 2210, 0, 357, 8190, 2, 5068, 1137, 2, 0, 8115, 0, 1, 0, 8192, 11, 0, 0, 0, 8186, 21, 7516] payload_rows_per_dst=[16376, 16330, 16382, 16384, 12404, 16232, 16384, 15058] dup_rows_per_dst=[46, 296, 4418, 714, 10, 0, 22, 16] combine_rows_per_src=[16376, 16330, 16382, 16384, 12404, 16232, 16384, 15058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124726 payload_rows_remote=108342 dup_rows=6346 dup_rows_remote=4894 dedup_ratio=0.048416 router_tokens_per_expert=[7, 0, 8, 218, 1, 2, 8190, 725, 0, 7340, 11, 296, 24, 0, 8192, 0, 4711, 5409, 1, 3, 8178, 0, 0, 0, 8188, 0, 13, 4, 0, 8108, 2, 5905] payload_rows_per_dst=[16366, 16384, 15044, 16384, 16004, 16356, 16378, 11810] dup_rows_per_dst=[316, 1452, 250, 48, 4244, 0, 32, 4] combine_rows_per_src=[16366, 16384, 15044, 16384, 16004, 16356, 16378, 11810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=781615104 dispatch_meta_bytes=423096 combine_payload_bytes=781615104 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=782038200 ingress_bytes=1486258176 link_bytes=1486258176 token_bytes=1563653304 weight_movement_bytes=704643072 total_nvlink_bytes=2268296376 weight_to_token_ratio=0.450639 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=27648 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.322425391e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=923009024 dispatch_meta_bytes=459968 combine_payload_bytes=923009024 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=923468992 ingress_bytes=1803812864 link_bytes=1803812864 token_bytes=1846478016 weight_movement_bytes=880803840 total_nvlink_bytes=2727281856 weight_to_token_ratio=0.477018 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.010059909e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=920666112 dispatch_meta_bytes=451008 combine_payload_bytes=920666112 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=921117120 ingress_bytes=1272987648 link_bytes=1272987648 token_bytes=1841783232 weight_movement_bytes=352321536 total_nvlink_bytes=2194104768 weight_to_token_ratio=0.191294 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.128199458e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129574 payload_rows_remote=113330 dup_rows=1498 dup_rows_remote=1442 dedup_ratio=0.011429 router_tokens_per_expert=[0, 1, 8105, 16, 8182, 3, 0, 29, 73, 191, 1, 29, 0, 7, 4, 4, 0, 0, 8158, 0, 28, 8153, 35, 5, 8089, 19, 8160, 10, 8162, 0, 0, 8072] payload_rows_per_dst=[16244, 16384, 16384, 16342, 16230, 15664, 16182, 16144] dup_rows_per_dst=[56, 130, 528, 8, 674, 48, 54, 0] combine_rows_per_src=[16244, 16384, 16384, 16342, 16230, 15664, 16182, 16144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=895533056 dispatch_meta_bytes=459360 combine_payload_bytes=895533056 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=895992416 ingress_bytes=1247854592 link_bytes=1247854592 token_bytes=1791525472 weight_movement_bytes=352321536 total_nvlink_bytes=2143847008 weight_to_token_ratio=0.196660 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.987464835e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=887537664 dispatch_meta_bytes=452944 combine_payload_bytes=887537664 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=887990608 ingress_bytes=1944502272 link_bytes=1944502272 token_bytes=1775528272 weight_movement_bytes=1056964608 total_nvlink_bytes=2832492880 weight_to_token_ratio=0.595296 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=27776 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.088840105e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129604 payload_rows_remote=113220 dup_rows=1468 dup_rows_remote=1458 dedup_ratio=0.011200 router_tokens_per_expert=[0, 0, 8192, 4, 8188, 0, 0, 11, 0, 3, 0, 22, 16, 0, 0, 0, 2, 0, 8168, 0, 1, 8192, 0, 685, 8192, 5, 8186, 0, 8178, 1, 71, 7419] payload_rows_per_dst=[16384, 16380, 16382, 16372, 16340, 16384, 16384, 14978] dup_rows_per_dst=[8, 18, 24, 32, 0, 1372, 10, 4] combine_rows_per_src=[16384, 16380, 16382, 16372, 16340, 16384, 16384, 14978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130048 payload_rows_remote=113668 dup_rows=1024 dup_rows_remote=694 dedup_ratio=0.007812 router_tokens_per_expert=[22, 181, 8172, 0, 8181, 0, 0, 60, 0, 35, 1, 134, 0, 0, 0, 0, 74, 3, 8118, 0, 0, 8191, 0, 31, 8148, 5, 8169, 0, 8185, 0, 163, 7663] payload_rows_per_dst=[16370, 16366, 16380, 16338, 16252, 16384, 16306, 15652] dup_rows_per_dst=[380, 116, 330, 0, 138, 60, 0, 0] combine_rows_per_src=[16370, 16366, 16380, 16338, 16252, 16384, 16306, 15652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=928399360 dispatch_meta_bytes=459088 combine_payload_bytes=928399360 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=928858448 ingress_bytes=1633042432 link_bytes=1633042432 token_bytes=1857257808 weight_movement_bytes=704643072 total_nvlink_bytes=2561900880 weight_to_token_ratio=0.379400 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.144355954e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123508 payload_rows_remote=107170 dup_rows=7564 dup_rows_remote=1066 dedup_ratio=0.057709 router_tokens_per_expert=[3, 0, 8111, 16, 8092, 0, 4, 2, 25, 7, 1, 101, 0, 0, 22, 0, 109, 4, 8057, 1, 12, 8158, 7, 2, 8183, 9, 8171, 18, 8171, 0, 166, 8084] payload_rows_per_dst=[15918, 16196, 13740, 16102, 16246, 12590, 16378, 16338] dup_rows_per_dst=[22, 274, 226, 8, 16, 22, 498, 6498] combine_rows_per_src=[15918, 16196, 13740, 16102, 16246, 12590, 16378, 16338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=927498240 dispatch_meta_bytes=458712 combine_payload_bytes=927498240 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=927956952 ingress_bytes=1632141312 link_bytes=1632141312 token_bytes=1855455192 weight_movement_bytes=704643072 total_nvlink_bytes=2560098264 weight_to_token_ratio=0.379768 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.139310058e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129924 payload_rows_remote=113568 dup_rows=1148 dup_rows_remote=1144 dedup_ratio=0.008759 router_tokens_per_expert=[3, 0, 8188, 0, 8172, 0, 4, 41, 2, 475, 0, 1, 9, 2, 0, 5, 46, 1, 8191, 0, 2, 8191, 0, 17, 8164, 0, 8164, 1, 8190, 2, 1, 7664] payload_rows_per_dst=[16376, 16384, 16384, 16356, 16384, 16382, 16330, 15328] dup_rows_per_dst=[6, 50, 952, 4, 92, 38, 0, 6] combine_rows_per_src=[16376, 16384, 16384, 16356, 16384, 16382, 16330, 15328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=931168256 dispatch_meta_bytes=457448 combine_payload_bytes=931168256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=931625704 ingress_bytes=1283489792 link_bytes=1283489792 token_bytes=1862793960 weight_movement_bytes=352321536 total_nvlink_bytes=2215115496 weight_to_token_ratio=0.189136 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.187007080e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123826 payload_rows_remote=113546 dup_rows=7246 dup_rows_remote=7026 dedup_ratio=0.055283 router_tokens_per_expert=[0, 0, 4, 0, 0, 0, 8084, 125, 0, 8127, 6, 2921, 0, 427, 8173, 11, 1918, 3332, 0, 0, 8182, 12, 0, 17, 8179, 0, 1, 0, 0, 8163, 50, 7804] payload_rows_per_dst=[16328, 16232, 16268, 16368, 10280, 16382, 16360, 15608] dup_rows_per_dst=[6, 186, 5840, 854, 220, 40, 0, 100] combine_rows_per_src=[16328, 16232, 16268, 16368, 10280, 16382, 16360, 15608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130524 payload_rows_remote=114144 dup_rows=548 dup_rows_remote=468 dedup_ratio=0.004181 router_tokens_per_expert=[72, 0, 8183, 1, 8135, 0, 0, 3, 0, 132, 0, 0, 13, 0, 0, 0, 0, 4, 8104, 0, 0, 8190, 0, 40, 8121, 7, 8192, 0, 8186, 0, 44, 8109] payload_rows_per_dst=[16366, 16274, 16376, 16384, 16216, 16380, 16242, 16286] dup_rows_per_dst=[146, 2, 260, 26, 0, 80, 14, 20] combine_rows_per_src=[16366, 16274, 16376, 16384, 16216, 16380, 16242, 16286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=877936640 dispatch_meta_bytes=432944 combine_payload_bytes=877936640 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=878369584 ingress_bytes=1582579712 link_bytes=1582579712 token_bytes=1756306224 weight_movement_bytes=704643072 total_nvlink_bytes=2460949296 weight_to_token_ratio=0.401207 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.861785786e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127162 payload_rows_remote=111032 dup_rows=3910 dup_rows_remote=3902 dedup_ratio=0.029831 router_tokens_per_expert=[9, 78, 8180, 8, 8056, 13, 0, 0, 1, 351, 0, 3, 55, 57, 0, 0, 0, 3, 7979, 0, 1, 8192, 1, 1450, 8128, 0, 8183, 0, 8043, 0, 693, 6052] payload_rows_per_dst=[16382, 16130, 16352, 16382, 15962, 16384, 16256, 13314] dup_rows_per_dst=[168, 8, 444, 208, 2, 2904, 0, 176] combine_rows_per_src=[16382, 16130, 16352, 16382, 15962, 16384, 16256, 13314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=930349056 dispatch_meta_bytes=458848 combine_payload_bytes=930349056 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930807904 ingress_bytes=1282670592 link_bytes=1282670592 token_bytes=1861156960 weight_movement_bytes=352321536 total_nvlink_bytes=2213478496 weight_to_token_ratio=0.189302 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.182419902e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117570 payload_rows_remote=101208 dup_rows=13502 dup_rows_remote=6330 dedup_ratio=0.103012 router_tokens_per_expert=[3584, 40, 0, 11, 15, 8123, 24, 42, 27, 8130, 2, 3, 3, 28, 8138, 47, 0, 51, 0, 1869, 8098, 8132, 3844, 1, 723, 6337, 14, 0, 30, 9, 8156, 55] payload_rows_per_dst=[16362, 16384, 14634, 16320, 16290, 7690, 13584, 16306] dup_rows_per_dst=[7172, 1948, 14, 34, 3746, 0, 564, 24] combine_rows_per_src=[16362, 16384, 14634, 16320, 16290, 7690, 13584, 16306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=930168832 dispatch_meta_bytes=482288 combine_payload_bytes=930168832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930651120 ingress_bytes=1282490368 link_bytes=1282490368 token_bytes=1860819952 weight_movement_bytes=352321536 total_nvlink_bytes=2213141488 weight_to_token_ratio=0.189337 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=17664 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.181410723e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117460 payload_rows_remote=101366 dup_rows=13612 dup_rows_remote=7334 dedup_ratio=0.103851 router_tokens_per_expert=[2376, 0, 1, 0, 1, 8184, 0, 14, 5, 8187, 0, 0, 6, 1, 8192, 0, 0, 0, 0, 1155, 8186, 8190, 7780, 0, 1, 5059, 0, 5, 0, 4, 8189, 0] payload_rows_per_dst=[11524, 16370, 16384, 16384, 8508, 15812, 16094, 16384] dup_rows_per_dst=[1418, 28, 0, 14, 2024, 3848, 6278, 2] combine_rows_per_src=[11524, 16370, 16384, 16384, 8508, 15812, 16094, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=935067648 dispatch_meta_bytes=458448 combine_payload_bytes=935067648 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=935526096 ingress_bytes=1992032256 link_bytes=1992032256 token_bytes=1870593744 weight_movement_bytes=1056964608 total_nvlink_bytes=2927558352 weight_to_token_ratio=0.565042 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.115454912e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=909574144 dispatch_meta_bytes=459736 combine_payload_bytes=909574144 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=910033880 ingress_bytes=1261895680 link_bytes=1261895680 token_bytes=1819608024 weight_movement_bytes=352321536 total_nvlink_bytes=2171929560 weight_to_token_ratio=0.193625 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.066089067e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127392 payload_rows_remote=111008 dup_rows=3680 dup_rows_remote=3304 dedup_ratio=0.028076 router_tokens_per_expert=[871, 4, 0, 0, 0, 8192, 0, 24, 0, 8189, 4, 187, 20, 0, 8168, 44, 0, 64, 1, 518, 7999, 8163, 7961, 3, 2, 6762, 0, 175, 1, 0, 8184, 0] payload_rows_per_dst=[16334, 16384, 16384, 16384, 16004, 15928, 13604, 16370] dup_rows_per_dst=[1742, 48, 376, 80, 1160, 0, 274, 0] combine_rows_per_src=[16334, 16384, 16384, 16384, 16004, 15928, 13604, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=829095936 dispatch_meta_bytes=430152 combine_payload_bytes=829095936 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=829526088 ingress_bytes=1181417472 link_bytes=1181417472 token_bytes=1658622024 weight_movement_bytes=352321536 total_nvlink_bytes=2010943560 weight_to_token_ratio=0.212418 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.615444695e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105070 payload_rows_remote=88696 dup_rows=26002 dup_rows_remote=25906 dedup_ratio=0.198380 router_tokens_per_expert=[1559, 0, 1, 6, 0, 8174, 12, 476, 46, 8103, 0, 667, 12, 0, 8183, 7, 2, 1, 0, 2247, 8170, 8179, 5417, 1, 19, 6020, 19, 18, 0, 0, 8173, 24] payload_rows_per_dst=[7952, 11766, 16368, 16318, 4498, 16342, 15452, 16374] dup_rows_per_dst=[0, 944, 1252, 22, 2, 16152, 7534, 96] combine_rows_per_src=[7952, 11766, 16368, 16318, 4498, 16342, 15452, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=830390272 dispatch_meta_bytes=434800 combine_payload_bytes=830390272 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=830825072 ingress_bytes=1182711808 link_bytes=1182711808 token_bytes=1661215344 weight_movement_bytes=352321536 total_nvlink_bytes=2013536880 weight_to_token_ratio=0.212087 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=26112 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.622692437e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128074 payload_rows_remote=111702 dup_rows=2998 dup_rows_remote=2998 dedup_ratio=0.022873 router_tokens_per_expert=[999, 0, 1, 0, 0, 8190, 1, 0, 2, 8187, 4, 7, 0, 0, 8186, 0, 0, 6, 0, 479, 8191, 8192, 8152, 3, 6, 6738, 2, 1, 1, 0, 8188, 0] payload_rows_per_dst=[16384, 16382, 16382, 16372, 16382, 16304, 13490, 16378] dup_rows_per_dst=[2000, 0, 18, 0, 970, 6, 4, 0] combine_rows_per_src=[16384, 16382, 16382, 16372, 16382, 16304, 13490, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=909377536 dispatch_meta_bytes=457248 combine_payload_bytes=909377536 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=909834784 ingress_bytes=1437859840 link_bytes=1437859840 token_bytes=1819212320 weight_movement_bytes=528482304 total_nvlink_bytes=2347694624 weight_to_token_ratio=0.290501 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.051414912e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=726597632 dispatch_meta_bytes=458408 combine_payload_bytes=726597632 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=727056040 ingress_bytes=1431240704 link_bytes=1431240704 token_bytes=1453653672 weight_movement_bytes=704643072 total_nvlink_bytes=2158296744 weight_to_token_ratio=0.484739 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.014350513e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130750 payload_rows_remote=114366 dup_rows=322 dup_rows_remote=186 dedup_ratio=0.002457 router_tokens_per_expert=[0, 0, 8123, 0, 8164, 6, 0, 10, 0, 5, 0, 0, 60, 14, 0, 0, 70, 4, 8186, 0, 0, 8183, 2, 5, 8189, 20, 8181, 0, 8150, 0, 388, 7776] payload_rows_per_dst=[16246, 16350, 16300, 16384, 16384, 16376, 16384, 16326] dup_rows_per_dst=[0, 10, 10, 126, 136, 4, 34, 2] combine_rows_per_src=[16246, 16350, 16300, 16384, 16384, 16376, 16384, 16326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=915062784 dispatch_meta_bytes=458800 combine_payload_bytes=915062784 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=915521584 ingress_bytes=1443545088 link_bytes=1443545088 token_bytes=1830584368 weight_movement_bytes=528482304 total_nvlink_bytes=2359066672 weight_to_token_ratio=0.288696 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.083249927e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122116 payload_rows_remote=106564 dup_rows=8956 dup_rows_remote=8350 dedup_ratio=0.068329 router_tokens_per_expert=[125, 0, 0, 0, 0, 8192, 4, 2, 0, 8137, 0, 30, 2, 0, 8186, 1, 0, 88, 14, 2214, 8154, 8163, 7776, 0, 2, 6210, 0, 45, 0, 4, 8187, 0] payload_rows_per_dst=[10760, 16384, 16316, 16376, 16384, 15552, 13962, 16382] dup_rows_per_dst=[194, 12, 18, 2, 4556, 606, 3568, 0] combine_rows_per_src=[10760, 16384, 16316, 16376, 16384, 15552, 13962, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121630 payload_rows_remote=105248 dup_rows=9442 dup_rows_remote=8992 dedup_ratio=0.072037 router_tokens_per_expert=[2263, 0, 0, 5, 0, 8188, 1, 227, 26, 8187, 0, 51, 0, 0, 8190, 7, 0, 18, 7, 2177, 8173, 8098, 3950, 0, 21, 7713, 0, 44, 0, 0, 8190, 0] payload_rows_per_dst=[16352, 16382, 16376, 16380, 16382, 7900, 15478, 16380] dup_rows_per_dst=[4380, 450, 152, 14, 4368, 0, 78, 0] combine_rows_per_src=[16352, 16382, 16376, 16380, 16382, 7900, 15478, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=100118 payload_rows_remote=83742 dup_rows=30954 dup_rows_remote=19198 dedup_ratio=0.236160 router_tokens_per_expert=[0, 8058, 7690, 8, 2852, 42, 32, 0, 3213, 49, 8142, 0, 8142, 0, 0, 25, 8156, 0, 6, 1656, 21, 437, 4, 2, 7, 3, 8124, 8130, 30, 48, 59, 600] payload_rows_per_dst=[16376, 10508, 16340, 15288, 16324, 12848, 11090, 1344] dup_rows_per_dst=[11756, 4394, 10556, 0, 3312, 806, 0, 130] combine_rows_per_src=[16376, 10508, 16340, 15288, 16324, 12848, 11090, 1344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113532 payload_rows_remote=97350 dup_rows=17540 dup_rows_remote=15672 dedup_ratio=0.133820 router_tokens_per_expert=[0, 8192, 7850, 4, 31, 0, 0, 0, 324, 0, 8190, 19, 8159, 49, 0, 1, 8190, 0, 0, 7023, 1, 35, 0, 0, 0, 5, 8187, 8086, 0, 2, 0, 1188] payload_rows_per_dst=[15700, 14506, 1644, 16384, 16350, 16384, 16182, 16382] dup_rows_per_dst=[8, 62, 634, 34, 12484, 72, 1868, 2378] combine_rows_per_src=[15700, 14506, 1644, 16384, 16350, 16384, 16182, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=936886272 dispatch_meta_bytes=458208 combine_payload_bytes=936886272 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=937344480 ingress_bytes=1641529344 link_bytes=1641529344 token_bytes=1874230752 weight_movement_bytes=704643072 total_nvlink_bytes=2578873824 weight_to_token_ratio=0.375964 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.191879118e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=872972288 dispatch_meta_bytes=459656 combine_payload_bytes=872972288 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=873431944 ingress_bytes=2282258432 link_bytes=2282258432 token_bytes=1746404232 weight_movement_bytes=1409286144 total_nvlink_bytes=3155690376 weight_to_token_ratio=0.806964 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=21504 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.277969456e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119488 payload_rows_remote=103110 dup_rows=11584 dup_rows_remote=3930 dedup_ratio=0.088379 router_tokens_per_expert=[1, 8171, 7467, 26, 409, 0, 0, 0, 3790, 49, 8177, 0, 8190, 2, 0, 2, 8192, 2, 0, 5140, 0, 1413, 1, 0, 0, 0, 8192, 6267, 0, 27, 0, 18] payload_rows_per_dst=[14986, 16384, 16378, 16380, 10712, 16384, 12534, 15730] dup_rows_per_dst=[2, 818, 7654, 8, 228, 2786, 0, 88] combine_rows_per_src=[14986, 16384, 16378, 16380, 10712, 16384, 12534, 15730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=862191616 dispatch_meta_bytes=456960 combine_payload_bytes=862191616 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=862648576 ingress_bytes=1566834688 link_bytes=1566834688 token_bytes=1724840192 weight_movement_bytes=704643072 total_nvlink_bytes=2429483264 weight_to_token_ratio=0.408527 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.773620224e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=686014464 dispatch_meta_bytes=411760 combine_payload_bytes=686014464 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=686426224 ingress_bytes=1566818304 link_bytes=1566818304 token_bytes=1372440688 weight_movement_bytes=880803840 total_nvlink_bytes=2253244528 weight_to_token_ratio=0.641779 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.773528480e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108640 payload_rows_remote=92270 dup_rows=22432 dup_rows_remote=15990 dedup_ratio=0.171143 router_tokens_per_expert=[83, 8097, 6966, 0, 3878, 3, 0, 0, 1098, 0, 8176, 2, 8163, 9, 0, 0, 8147, 29, 13, 1182, 0, 38, 16, 11, 10, 1, 8164, 8191, 8, 27, 138, 3086] payload_rows_per_dst=[10862, 12482, 5366, 16146, 16384, 14646, 16384, 16370] dup_rows_per_dst=[70, 3424, 0, 18, 2416, 72, 9990, 6442] combine_rows_per_src=[10862, 12482, 5366, 16146, 16384, 14646, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=797491200 dispatch_meta_bytes=452088 combine_payload_bytes=797491200 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=797943288 ingress_bytes=1854455808 link_bytes=1854455808 token_bytes=1595434488 weight_movement_bytes=1056964608 total_nvlink_bytes=2652399096 weight_to_token_ratio=0.662493 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.038417844e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=844677120 dispatch_meta_bytes=428160 combine_payload_bytes=844677120 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=845105280 ingress_bytes=1549320192 link_bytes=1549320192 token_bytes=1689782400 weight_movement_bytes=704643072 total_nvlink_bytes=2394425472 weight_to_token_ratio=0.417002 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=25344 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.675546357e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122150 payload_rows_remote=105774 dup_rows=8922 dup_rows_remote=8718 dedup_ratio=0.068069 router_tokens_per_expert=[0, 8188, 7947, 0, 399, 1, 10, 0, 3890, 11, 8187, 13, 8113, 164, 8, 5, 8187, 1, 3, 4306, 0, 14, 0, 2, 2, 0, 8192, 7872, 1, 6, 0, 14] payload_rows_per_dst=[15894, 16378, 16376, 16376, 8618, 16382, 15748, 16378] dup_rows_per_dst=[0, 820, 7826, 204, 2, 32, 0, 38] combine_rows_per_src=[15894, 16378, 16376, 16376, 8618, 16382, 15748, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=755875840 dispatch_meta_bytes=433040 combine_payload_bytes=755875840 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=756308880 ingress_bytes=1108197376 link_bytes=1108197376 token_bytes=1512184720 weight_movement_bytes=352321536 total_nvlink_bytes=1864506256 weight_to_token_ratio=0.232988 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=28544 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.205442721e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121348 payload_rows_remote=104968 dup_rows=9724 dup_rows_remote=9562 dedup_ratio=0.074188 router_tokens_per_expert=[0, 8172, 7584, 2, 539, 9, 0, 0, 132, 18, 8192, 3, 8149, 6, 2, 6, 8144, 0, 0, 5345, 0, 99, 0, 0, 0, 3, 8183, 8082, 0, 30, 18, 2818] payload_rows_per_dst=[15172, 16384, 8232, 16298, 16332, 16380, 16170, 16380] dup_rows_per_dst=[0, 1078, 266, 28, 2550, 162, 0, 5640] combine_rows_per_src=[15172, 16384, 8232, 16298, 16332, 16380, 16170, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=866500608 dispatch_meta_bytes=457968 combine_payload_bytes=866500608 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=866958576 ingress_bytes=1394982912 link_bytes=1394982912 token_bytes=1733459184 weight_movement_bytes=528482304 total_nvlink_bytes=2261941488 weight_to_token_ratio=0.304872 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.811322013e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124322 payload_rows_remote=107940 dup_rows=6750 dup_rows_remote=3334 dedup_ratio=0.051498 router_tokens_per_expert=[1056, 1, 0, 0, 1, 8191, 2, 601, 18, 8180, 1, 0, 13, 0, 8173, 0, 0, 0, 4, 1707, 8188, 8192, 7466, 0, 526, 5028, 0, 0, 0, 0, 8188, 0] payload_rows_per_dst=[16384, 16384, 16384, 16372, 16382, 14932, 11108, 16376] dup_rows_per_dst=[2114, 1206, 14, 0, 3416, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16372, 16382, 14932, 11108, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113788 payload_rows_remote=97476 dup_rows=17284 dup_rows_remote=16736 dedup_ratio=0.131866 router_tokens_per_expert=[8, 8114, 8131, 291, 16, 0, 24, 0, 18, 61, 8040, 0, 8156, 33, 23, 83, 8124, 8148, 5, 4, 30, 2127, 118, 3077, 1407, 1389, 10, 30, 0, 32, 8019, 18] payload_rows_per_dst=[16312, 16324, 16194, 16358, 16312, 10512, 5654, 16122] dup_rows_per_dst=[548, 2, 44, 214, 16250, 192, 18, 16] combine_rows_per_src=[16312, 16324, 16194, 16358, 16312, 10512, 5654, 16122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120236 payload_rows_remote=103892 dup_rows=10836 dup_rows_remote=10832 dedup_ratio=0.082672 router_tokens_per_expert=[0, 8155, 6935, 6, 2, 0, 17, 0, 1679, 87, 8164, 16, 8176, 426, 0, 0, 8190, 3, 0, 4227, 0, 674, 0, 2, 0, 0, 8192, 8145, 1, 3, 0, 2436] payload_rows_per_dst=[13874, 16344, 16360, 16358, 12630, 16384, 16290, 11996] dup_rows_per_dst=[8, 4, 3532, 846, 4316, 1352, 0, 778] combine_rows_per_src=[13874, 16344, 16360, 16358, 12630, 16384, 16290, 11996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115148 payload_rows_remote=98764 dup_rows=15924 dup_rows_remote=14126 dedup_ratio=0.121490 router_tokens_per_expert=[0, 8165, 8192, 248, 0, 0, 0, 0, 0, 0, 8134, 467, 8192, 0, 0, 0, 8187, 8192, 0, 0, 0, 6673, 6, 6, 875, 0, 24, 0, 6, 1, 8168, 0] payload_rows_per_dst=[16384, 16384, 16292, 16384, 2480, 14492, 16384, 16348] dup_rows_per_dst=[442, 0, 910, 0, 0, 12772, 1798, 2] combine_rows_per_src=[16384, 16384, 16292, 16384, 2480, 14492, 16384, 16348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 3, 2, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=859897856 dispatch_meta_bytes=458120 combine_payload_bytes=859897856 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=860355976 ingress_bytes=1212219392 link_bytes=1212219392 token_bytes=1720253832 weight_movement_bytes=352321536 total_nvlink_bytes=2072575368 weight_to_token_ratio=0.204808 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.787922590e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124902 payload_rows_remote=108628 dup_rows=6170 dup_rows_remote=6170 dedup_ratio=0.047073 router_tokens_per_expert=[6, 8186, 8191, 514, 0, 0, 0, 0, 0, 0, 8137, 0, 8138, 0, 0, 0, 8179, 8188, 0, 0, 13, 4874, 246, 151, 1199, 1336, 24, 0, 0, 10, 8144, 0] payload_rows_per_dst=[16384, 16382, 16274, 16276, 16358, 10536, 16384, 16308] dup_rows_per_dst=[1028, 0, 0, 0, 0, 32, 5110, 0] combine_rows_per_src=[16384, 16382, 16274, 16276, 16358, 10536, 16384, 16308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=884244480 dispatch_meta_bytes=445096 combine_payload_bytes=884244480 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=884689576 ingress_bytes=1236566016 link_bytes=1236566016 token_bytes=1768934056 weight_movement_bytes=352321536 total_nvlink_bytes=2121255592 weight_to_token_ratio=0.199172 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=28032 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.924253522e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113012 payload_rows_remote=96628 dup_rows=18060 dup_rows_remote=16864 dedup_ratio=0.137787 router_tokens_per_expert=[18, 8132, 8153, 381, 1, 1, 0, 1, 0, 0, 8189, 252, 8171, 2, 17, 0, 8175, 8173, 0, 2, 56, 4436, 804, 1735, 603, 0, 4, 9, 32, 0, 8176, 13] payload_rows_per_dst=[12606, 16312, 14994, 16376, 3588, 16384, 16368, 16384] dup_rows_per_dst=[756, 78, 648, 144, 0, 14028, 1210, 1196] combine_rows_per_src=[12606, 16312, 14994, 16376, 3588, 16384, 16368, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=798523392 dispatch_meta_bytes=456848 combine_payload_bytes=798523392 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=798980240 ingress_bytes=2560131072 link_bytes=2560131072 token_bytes=1597503632 weight_movement_bytes=1761607680 total_nvlink_bytes=3359111312 weight_to_token_ratio=1.102725 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.433566535e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=851083264 dispatch_meta_bytes=458896 combine_payload_bytes=851083264 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=851542160 ingress_bytes=1203404800 link_bytes=1203404800 token_bytes=1702625424 weight_movement_bytes=352321536 total_nvlink_bytes=2054946960 weight_to_token_ratio=0.206928 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.738564554e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=809074688 dispatch_meta_bytes=451560 combine_payload_bytes=809074688 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=809526248 ingress_bytes=1161396224 link_bytes=1161396224 token_bytes=1618600936 weight_movement_bytes=352321536 total_nvlink_bytes=1970922472 weight_to_token_ratio=0.217670 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=26624 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.503334064e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127038 payload_rows_remote=110714 dup_rows=4034 dup_rows_remote=4032 dedup_ratio=0.030777 router_tokens_per_expert=[0, 8179, 8188, 0, 7, 3, 0, 2, 11, 2, 8167, 4, 8155, 2, 0, 6, 8185, 8190, 0, 3, 0, 5537, 19, 719, 1961, 4, 12, 1, 0, 0, 8179, 0] payload_rows_per_dst=[16358, 16384, 16342, 16324, 16376, 12514, 16382, 16358] dup_rows_per_dst=[0, 16, 26, 2, 0, 36, 3954, 0] combine_rows_per_src=[16358, 16384, 16342, 16324, 16376, 12514, 16382, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=889880576 dispatch_meta_bytes=459192 combine_payload_bytes=889880576 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=890339768 ingress_bytes=1418362880 link_bytes=1418362880 token_bytes=1780220344 weight_movement_bytes=528482304 total_nvlink_bytes=2308702648 weight_to_token_ratio=0.296863 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.942240074e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=791576576 dispatch_meta_bytes=453968 combine_payload_bytes=791576576 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=792030544 ingress_bytes=1143898112 link_bytes=1143898112 token_bytes=1583607120 weight_movement_bytes=352321536 total_nvlink_bytes=1935928656 weight_to_token_ratio=0.222480 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.405351941e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=906969088 dispatch_meta_bytes=458984 combine_payload_bytes=906969088 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=907428072 ingress_bytes=1259290624 link_bytes=1259290624 token_bytes=1814397160 weight_movement_bytes=352321536 total_nvlink_bytes=2166718696 weight_to_token_ratio=0.194181 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.051501841e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121880 payload_rows_remote=114168 dup_rows=9192 dup_rows_remote=9048 dedup_ratio=0.070129 router_tokens_per_expert=[5, 8176, 8169, 276, 0, 0, 0, 0, 0, 1, 8064, 42, 8169, 0, 0, 0, 8192, 8185, 3, 0, 16, 3803, 18, 41, 4236, 13, 0, 0, 13, 0, 8114, 0] payload_rows_per_dst=[16384, 16338, 16214, 16338, 16284, 7712, 16370, 16240] dup_rows_per_dst=[530, 0, 0, 0, 6, 144, 8498, 14] combine_rows_per_src=[16384, 16338, 16214, 16338, 16284, 7712, 16370, 16240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121358 payload_rows_remote=113004 dup_rows=9714 dup_rows_remote=9654 dedup_ratio=0.074112 router_tokens_per_expert=[0, 8190, 8182, 0, 392, 0, 170, 0, 1797, 24, 8174, 0, 8169, 173, 9, 1, 8192, 0, 15, 4151, 0, 0, 0, 1, 0, 0, 8181, 7408, 6, 0, 0, 2301] payload_rows_per_dst=[16364, 16378, 16302, 16380, 8354, 16380, 14816, 16384] dup_rows_per_dst=[0, 1108, 3606, 324, 60, 2, 0, 4614] combine_rows_per_src=[16364, 16378, 16302, 16380, 8354, 16380, 14816, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112008 payload_rows_remote=95628 dup_rows=19064 dup_rows_remote=1496 dedup_ratio=0.145447 router_tokens_per_expert=[45, 8136, 8138, 7840, 37, 0, 156, 6214, 15, 0, 8162, 5, 8071, 0, 1, 2, 17, 0, 7918, 330, 26, 1, 4, 29, 163, 163, 9, 8033, 36, 16, 1969, 0] payload_rows_per_dst=[16380, 12536, 16354, 16148, 15762, 16246, 14540, 4042] dup_rows_per_dst=[17568, 278, 422, 578, 34, 10, 174, 0] combine_rows_per_src=[16380, 12536, 16354, 16148, 15762, 16246, 14540, 4042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119576 payload_rows_remote=103192 dup_rows=11496 dup_rows_remote=11446 dedup_ratio=0.087708 router_tokens_per_expert=[0, 8191, 8187, 42, 23, 0, 0, 7, 0, 4, 8191, 171, 8134, 0, 0, 0, 8190, 8186, 0, 5, 3, 7909, 0, 335, 110, 0, 0, 1, 0, 0, 7845, 2] payload_rows_per_dst=[16382, 16384, 16384, 16268, 16384, 16280, 5800, 15694] dup_rows_per_dst=[84, 50, 348, 0, 10794, 214, 6, 0] combine_rows_per_src=[16382, 16384, 16384, 16268, 16384, 16280, 5800, 15694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119616 payload_rows_remote=103238 dup_rows=11456 dup_rows_remote=11454 dedup_ratio=0.087402 router_tokens_per_expert=[0, 8190, 8188, 8188, 0, 2, 5, 904, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 1, 0, 0, 8189, 17, 100, 7176, 0] payload_rows_per_dst=[6100, 16380, 16384, 16382, 16382, 16380, 16378, 15230] dup_rows_per_dst=[0, 1820, 0, 0, 0, 24, 2, 9610] combine_rows_per_src=[6100, 16380, 16384, 16382, 16382, 16380, 16378, 15230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 3, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=935264256 dispatch_meta_bytes=492864 combine_payload_bytes=935264256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935757120 ingress_bytes=1287585792 link_bytes=1287585792 token_bytes=1871021376 weight_movement_bytes=352321536 total_nvlink_bytes=2223342912 weight_to_token_ratio=0.188304 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=22144 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.209942970e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128480 payload_rows_remote=112154 dup_rows=2592 dup_rows_remote=2568 dedup_ratio=0.019775 router_tokens_per_expert=[0, 8108, 8186, 8185, 4, 5, 3, 1085, 0, 13, 8161, 1, 8187, 0, 1, 0, 0, 0, 8165, 72, 105, 0, 0, 0, 27, 0, 0, 8187, 0, 1861, 5158, 22] payload_rows_per_dst=[16216, 16384, 16326, 16376, 16340, 16384, 16374, 14080] dup_rows_per_dst=[0, 2180, 24, 0, 134, 198, 54, 2] combine_rows_per_src=[16216, 16384, 16326, 16376, 16340, 16384, 16374, 14080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=925728768 dispatch_meta_bytes=490632 combine_payload_bytes=925728768 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=926219400 ingress_bytes=1630371840 link_bytes=1630371840 token_bytes=1851948168 weight_movement_bytes=704643072 total_nvlink_bytes=2556591240 weight_to_token_ratio=0.380487 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=21248 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.129401753e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113282 payload_rows_remote=97004 dup_rows=17790 dup_rows_remote=8936 dedup_ratio=0.135727 router_tokens_per_expert=[0, 8070, 8152, 7895, 31, 64, 16, 3619, 10, 0, 8165, 0, 8127, 3, 6, 0, 0, 151, 7941, 60, 0, 1, 1, 17, 472, 46, 0, 8141, 0, 1098, 3449, 1] payload_rows_per_dst=[0, 16274, 16186, 15652, 16176, 16340, 16376, 16278] dup_rows_per_dst=[0, 7080, 0, 10, 292, 2, 1552, 8854] combine_rows_per_src=[0, 16274, 16186, 15652, 16176, 16340, 16376, 16278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=783384576 dispatch_meta_bytes=388496 combine_payload_bytes=783384576 prefetch_bytes=352321536 reduce_bytes=1233125376 egress_bytes=783773072 ingress_bytes=2368831488 link_bytes=2368831488 token_bytes=1567157648 weight_movement_bytes=1585446912 total_nvlink_bytes=3152604560 weight_to_token_ratio=1.011670 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=25728 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.326446753e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129664 payload_rows_remote=113336 dup_rows=1408 dup_rows_remote=920 dedup_ratio=0.010742 router_tokens_per_expert=[5, 7507, 8181, 8173, 3, 18, 3, 19, 0, 38, 8176, 19, 8139, 1, 268, 0, 0, 2, 8186, 47, 0, 0, 20, 7, 1, 0, 11, 8185, 0, 329, 8179, 19] payload_rows_per_dst=[15138, 16310, 16362, 16328, 16378, 16384, 16382, 16382] dup_rows_per_dst=[0, 8, 104, 488, 92, 32, 12, 672] combine_rows_per_src=[15138, 16310, 16362, 16328, 16378, 16384, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=845725696 dispatch_meta_bytes=458768 combine_payload_bytes=845725696 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=846184464 ingress_bytes=1550368768 link_bytes=1550368768 token_bytes=1691910160 weight_movement_bytes=704643072 total_nvlink_bytes=2396553232 weight_to_token_ratio=0.416478 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.681417945e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=845348864 dispatch_meta_bytes=458552 combine_payload_bytes=845348864 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=845807416 ingress_bytes=1373831168 link_bytes=1373831168 token_bytes=1691156280 weight_movement_bytes=528482304 total_nvlink_bytes=2219638584 weight_to_token_ratio=0.312498 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.692881075e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=918765568 dispatch_meta_bytes=458888 combine_payload_bytes=918765568 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=919224456 ingress_bytes=1271087104 link_bytes=1271087104 token_bytes=1837990024 weight_movement_bytes=352321536 total_nvlink_bytes=2190311560 weight_to_token_ratio=0.191688 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.117557205e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=794656768 dispatch_meta_bytes=423760 combine_payload_bytes=794656768 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=795080528 ingress_bytes=1146978304 link_bytes=1146978304 token_bytes=1589737296 weight_movement_bytes=352321536 total_nvlink_bytes=1942058832 weight_to_token_ratio=0.221622 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=23936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.422599730e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=928448512 dispatch_meta_bytes=457024 combine_payload_bytes=928448512 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=928905536 ingress_bytes=1456930816 link_bytes=1456930816 token_bytes=1857354048 weight_movement_bytes=528482304 total_nvlink_bytes=2385836352 weight_to_token_ratio=0.284535 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.158204417e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129318 payload_rows_remote=112936 dup_rows=1754 dup_rows_remote=1754 dedup_ratio=0.013382 router_tokens_per_expert=[0, 8191, 8191, 8049, 1, 0, 0, 976, 0, 0, 8190, 0, 8192, 0, 0, 1, 6, 13, 8160, 39, 0, 0, 0, 0, 0, 1, 0, 8184, 4, 527, 6811, 0] payload_rows_per_dst=[16382, 16358, 16380, 16384, 16382, 16382, 16370, 14680] dup_rows_per_dst=[0, 1694, 0, 2, 54, 0, 0, 4] combine_rows_per_src=[16382, 16358, 16380, 16384, 16382, 16382, 16370, 14680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129744 payload_rows_remote=113504 dup_rows=1328 dup_rows_remote=1328 dedup_ratio=0.010132 router_tokens_per_expert=[0, 8179, 8190, 258, 0, 14, 3, 2, 6, 3, 8192, 0, 8183, 0, 0, 11, 8116, 8182, 4, 0, 0, 7609, 3, 0, 66, 314, 0, 6, 0, 3, 8192, 0] payload_rows_per_dst=[16366, 16384, 16384, 16380, 16240, 15224, 16382, 16384] dup_rows_per_dst=[508, 34, 18, 8, 0, 0, 754, 6] combine_rows_per_src=[16366, 16384, 16384, 16380, 16240, 15224, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113974 payload_rows_remote=97650 dup_rows=17098 dup_rows_remote=16272 dedup_ratio=0.130447 router_tokens_per_expert=[52, 63, 10, 0, 8127, 25, 7, 24, 56, 97, 8052, 7825, 76, 1064, 41, 8, 6427, 29, 10, 0, 8126, 38, 18, 0, 4113, 34, 0, 4128, 8099, 0, 873, 8114] payload_rows_per_dst=[16324, 16332, 15912, 14844, 13796, 16228, 9200, 11338] dup_rows_per_dst=[826, 154, 1680, 2184, 7362, 62, 4584, 246] combine_rows_per_src=[16324, 16332, 15912, 14844, 13796, 16228, 9200, 11338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126528 payload_rows_remote=110654 dup_rows=4544 dup_rows_remote=4410 dedup_ratio=0.034668 router_tokens_per_expert=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 39, 8192, 8186, 5, 302, 3, 0, 8107, 0, 3, 2, 8189, 0, 0, 1, 7506, 0, 0, 498, 8124, 0, 14, 8174] payload_rows_per_dst=[16384, 16382, 16374, 12646, 16214, 16380, 15874, 16274] dup_rows_per_dst=[0, 0, 76, 502, 10, 0, 134, 3822] combine_rows_per_src=[16384, 16382, 16374, 12646, 16214, 16380, 15874, 16274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119630 payload_rows_remote=113578 dup_rows=11442 dup_rows_remote=11304 dedup_ratio=0.087296 router_tokens_per_expert=[0, 8078, 8153, 8184, 0, 0, 9, 417, 1, 7, 8029, 4, 8170, 0, 0, 0, 1, 46, 8189, 26, 11, 0, 0, 0, 37, 10, 1, 8177, 0, 2314, 5628, 44] payload_rows_per_dst=[16376, 6052, 16072, 16340, 16382, 16320, 16374, 15714] dup_rows_per_dst=[10810, 138, 10, 0, 142, 8, 76, 258] combine_rows_per_src=[16376, 6052, 16072, 16340, 16382, 16320, 16374, 15714] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=925171712 dispatch_meta_bytes=458760 combine_payload_bytes=925171712 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=925630472 ingress_bytes=1453654016 link_bytes=1453654016 token_bytes=1850802184 weight_movement_bytes=528482304 total_nvlink_bytes=2379284488 weight_to_token_ratio=0.285542 gemm_rows_defined=66688 gemm_rows_charged=67584 gemm_tail_pad_rows=896 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.013258 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.139855705e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127552 payload_rows_remote=111294 dup_rows=3520 dup_rows_remote=3394 dedup_ratio=0.026855 router_tokens_per_expert=[18, 1, 0, 0, 8183, 1, 0, 9, 91, 0, 8188, 8101, 0, 163, 0, 5, 7628, 0, 0, 15, 8192, 0, 0, 48, 8088, 0, 11, 459, 8182, 0, 55, 8098] payload_rows_per_dst=[16384, 16384, 16258, 16196, 15280, 16384, 14302, 16364] dup_rows_per_dst=[30, 2, 126, 336, 2004, 96, 816, 110] combine_rows_per_src=[16384, 16384, 16258, 16196, 15280, 16384, 14302, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=929824768 dispatch_meta_bytes=459328 combine_payload_bytes=929824768 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=930284096 ingress_bytes=1986789376 link_bytes=1986789376 token_bytes=1860108864 weight_movement_bytes=1056964608 total_nvlink_bytes=2917073472 weight_to_token_ratio=0.568227 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=25856 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.112519118e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=799948800 dispatch_meta_bytes=455688 combine_payload_bytes=799948800 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=800404488 ingress_bytes=1504591872 link_bytes=1504591872 token_bytes=1600353288 weight_movement_bytes=704643072 total_nvlink_bytes=2304996360 weight_to_token_ratio=0.440305 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.017045 gemm_active_groups=13 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.425086436e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120592 payload_rows_remote=104678 dup_rows=10480 dup_rows_remote=2806 dedup_ratio=0.079956 router_tokens_per_expert=[50, 69, 0, 0, 8116, 4, 3, 1, 3, 15, 8108, 8116, 71, 399, 0, 2, 7770, 3, 7, 3, 8149, 27, 0, 0, 6504, 31, 35, 2201, 7941, 0, 19, 7889] payload_rows_per_dst=[16362, 16172, 15068, 8856, 15548, 16348, 16324, 15914] dup_rows_per_dst=[128, 2, 2, 222, 18, 18, 2416, 7674] combine_rows_per_src=[16362, 16172, 15068, 8856, 15548, 16348, 16324, 15914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121514 payload_rows_remote=105130 dup_rows=9558 dup_rows_remote=1428 dedup_ratio=0.072922 router_tokens_per_expert=[6, 0, 0, 0, 8189, 0, 46, 1, 80, 0, 8168, 8186, 1, 4053, 3, 10, 7358, 2, 0, 0, 8189, 1, 2, 23, 8166, 0, 0, 546, 4397, 20, 0, 8089] payload_rows_per_dst=[16146, 16382, 16374, 16384, 14720, 16378, 16332, 8798] dup_rows_per_dst=[0, 90, 158, 8130, 0, 52, 1092, 36] combine_rows_per_src=[16146, 16382, 16374, 16384, 14720, 16378, 16332, 8798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=930430976 dispatch_meta_bytes=499528 combine_payload_bytes=930430976 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930930504 ingress_bytes=1282752512 link_bytes=1282752512 token_bytes=1861361480 weight_movement_bytes=352321536 total_nvlink_bytes=2213683016 weight_to_token_ratio=0.189282 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=26112 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.182878620e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=906477568 dispatch_meta_bytes=460256 combine_payload_bytes=906477568 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=906937824 ingress_bytes=1611120640 link_bytes=1611120640 token_bytes=1813415392 weight_movement_bytes=704643072 total_nvlink_bytes=2518058464 weight_to_token_ratio=0.388572 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=25344 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.021603069e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=911720448 dispatch_meta_bytes=458752 combine_payload_bytes=911720448 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=912179200 ingress_bytes=1968685056 link_bytes=1968685056 token_bytes=1823899648 weight_movement_bytes=1056964608 total_nvlink_bytes=2880864256 weight_to_token_ratio=0.579508 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.102381454e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=861224960 dispatch_meta_bytes=426232 combine_payload_bytes=861224960 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=861651192 ingress_bytes=1213546496 link_bytes=1213546496 token_bytes=1722876152 weight_movement_bytes=352321536 total_nvlink_bytes=2075197688 weight_to_token_ratio=0.204496 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.795353818e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=857522176 dispatch_meta_bytes=429936 combine_payload_bytes=857522176 prefetch_bytes=176160768 reduce_bytes=1233125376 egress_bytes=857952112 ingress_bytes=2266808320 link_bytes=2266808320 token_bytes=1715474288 weight_movement_bytes=1409286144 total_nvlink_bytes=3124760432 weight_to_token_ratio=0.821514 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.269318038e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128650 payload_rows_remote=112322 dup_rows=2422 dup_rows_remote=2422 dedup_ratio=0.018478 router_tokens_per_expert=[4, 3, 3, 0, 8176, 6, 0, 0, 43, 31, 7861, 8080, 36, 2, 0, 0, 7524, 0, 0, 2, 8164, 0, 0, 0, 7751, 1, 0, 800, 8186, 0, 702, 8161] payload_rows_per_dst=[15708, 16364, 16258, 16384, 15052, 16328, 16174, 16382] dup_rows_per_dst=[4, 0, 50, 44, 0, 0, 930, 1394] combine_rows_per_src=[15708, 16364, 16258, 16384, 15052, 16328, 16174, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108170 payload_rows_remote=91786 dup_rows=22902 dup_rows_remote=22664 dedup_ratio=0.174728 router_tokens_per_expert=[46, 47, 0, 30, 6, 14, 42, 33, 8141, 7761, 8027, 8, 67, 7811, 3, 4, 18, 4, 8162, 252, 8125, 0, 0, 13, 40, 384, 0, 8141, 8110, 213, 0, 34] payload_rows_per_dst=[16384, 16168, 16242, 16384, 11038, 16276, 2894, 12784] dup_rows_per_dst=[238, 430, 5660, 15668, 778, 90, 14, 24] combine_rows_per_src=[16384, 16168, 16242, 16384, 11038, 16276, 2894, 12784] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120760 payload_rows_remote=104380 dup_rows=10312 dup_rows_remote=1256 dedup_ratio=0.078674 router_tokens_per_expert=[0, 65, 7, 0, 6, 0, 1, 0, 8192, 8177, 8131, 0, 0, 3060, 0, 0, 0, 0, 8191, 605, 8192, 0, 0, 0, 0, 3470, 1064, 8184, 8191, 0, 0, 0] payload_rows_per_dst=[16376, 16380, 16354, 6120, 16384, 16384, 16380, 16382] dup_rows_per_dst=[34, 14, 0, 0, 1208, 0, 9056, 0] combine_rows_per_src=[16376, 16380, 16354, 6120, 16384, 16384, 16380, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129510 payload_rows_remote=113138 dup_rows=1562 dup_rows_remote=1412 dedup_ratio=0.011917 router_tokens_per_expert=[9, 8039, 8170, 8156, 0, 0, 0, 637, 2, 30, 8192, 0, 8182, 10, 0, 0, 0, 104, 8157, 0, 5, 0, 6, 0, 1, 17, 2, 8184, 23, 0, 7602, 8] payload_rows_per_dst=[16102, 16314, 16384, 16384, 16372, 16358, 16384, 15212] dup_rows_per_dst=[0, 1266, 64, 0, 150, 4, 24, 54] combine_rows_per_src=[16102, 16314, 16384, 16384, 16372, 16358, 16384, 15212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121690 payload_rows_remote=105326 dup_rows=9382 dup_rows_remote=9382 dedup_ratio=0.071579 router_tokens_per_expert=[2, 0, 0, 13, 8182, 0, 0, 0, 1, 13, 8190, 8088, 34, 437, 0, 2, 5168, 1, 0, 0, 8191, 2, 8, 0, 8158, 0, 0, 2619, 8165, 0, 83, 8179] payload_rows_per_dst=[16372, 16364, 16202, 16382, 16216, 16384, 7386, 16384] dup_rows_per_dst=[20, 0, 2, 940, 8146, 18, 144, 112] combine_rows_per_src=[16372, 16364, 16202, 16382, 16216, 16384, 7386, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=920141824 dispatch_meta_bytes=458976 combine_payload_bytes=920141824 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=920600800 ingress_bytes=1448624128 link_bytes=1448624128 token_bytes=1840742624 weight_movement_bytes=528482304 total_nvlink_bytes=2369224928 weight_to_token_ratio=0.287103 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.111690431e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121874 payload_rows_remote=105498 dup_rows=9198 dup_rows_remote=9198 dedup_ratio=0.070175 router_tokens_per_expert=[0, 0, 0, 0, 23, 0, 0, 0, 8185, 8188, 8188, 0, 0, 5430, 1, 0, 73, 0, 8186, 1152, 8192, 3, 1, 0, 0, 5045, 244, 4365, 8192, 24, 44, 0] payload_rows_per_dst=[16380, 16382, 16376, 13494, 16376, 16384, 10098, 16384] dup_rows_per_dst=[0, 30, 0, 6098, 2446, 8, 480, 136] combine_rows_per_src=[16380, 16382, 16376, 13494, 16376, 16384, 10098, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=751910912 dispatch_meta_bytes=457800 combine_payload_bytes=751910912 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=752368712 ingress_bytes=1280393216 link_bytes=1280393216 token_bytes=1504279624 weight_movement_bytes=528482304 total_nvlink_bytes=2032761928 weight_to_token_ratio=0.351319 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.015152 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.169667547e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[3, 3, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=855080960 dispatch_meta_bytes=422544 combine_payload_bytes=855080960 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=855503504 ingress_bytes=1559724032 link_bytes=1559724032 token_bytes=1710584464 weight_movement_bytes=704643072 total_nvlink_bytes=2415227536 weight_to_token_ratio=0.411931 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=19584 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.733803518e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123752 payload_rows_remote=108192 dup_rows=7320 dup_rows_remote=3296 dedup_ratio=0.055847 router_tokens_per_expert=[2, 15, 5, 1, 15, 0, 5, 5, 7841, 5340, 8181, 0, 9, 2351, 3, 3, 0, 0, 8176, 213, 8180, 1, 0, 1, 1, 8160, 89, 8116, 8159, 0, 663, 1] payload_rows_per_dst=[16378, 15704, 10680, 15560, 16352, 16364, 16336, 16378] dup_rows_per_dst=[34, 24, 0, 4024, 426, 0, 1544, 1268] combine_rows_per_src=[16378, 15704, 10680, 15560, 16352, 16364, 16336, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=926826496 dispatch_meta_bytes=458200 combine_payload_bytes=926826496 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=927284696 ingress_bytes=1807630336 link_bytes=1807630336 token_bytes=1854111192 weight_movement_bytes=880803840 total_nvlink_bytes=2734915032 weight_to_token_ratio=0.475054 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.012197534e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113432 payload_rows_remote=97048 dup_rows=17640 dup_rows_remote=16634 dedup_ratio=0.134583 router_tokens_per_expert=[4, 14, 19, 0, 509, 0, 25, 2, 8187, 8162, 8060, 6, 0, 2302, 1, 31, 164, 14, 8159, 532, 8176, 0, 64, 0, 9, 4321, 265, 7992, 8172, 75, 239, 32] payload_rows_per_dst=[16070, 15600, 16336, 4666, 12022, 16352, 16002, 16384] dup_rows_per_dst=[66, 1058, 846, 2, 1176, 410, 13076, 1006] combine_rows_per_src=[16070, 15600, 16336, 4666, 12022, 16352, 16002, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=862830592 dispatch_meta_bytes=458832 combine_payload_bytes=862830592 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=863289424 ingress_bytes=1391312896 link_bytes=1391312896 token_bytes=1726120016 weight_movement_bytes=528482304 total_nvlink_bytes=2254602320 weight_to_token_ratio=0.306168 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.790771455e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=864239616 dispatch_meta_bytes=458784 combine_payload_bytes=864239616 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=864698400 ingress_bytes=2273525760 link_bytes=2273525760 token_bytes=1728938016 weight_movement_bytes=1409286144 total_nvlink_bytes=3138224160 weight_to_token_ratio=0.815117 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.273079524e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=886308864 dispatch_meta_bytes=445952 combine_payload_bytes=886308864 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=886754816 ingress_bytes=1238630400 link_bytes=1238630400 token_bytes=1773063680 weight_movement_bytes=352321536 total_nvlink_bytes=2125385216 weight_to_token_ratio=0.198708 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=18688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.935813210e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124810 payload_rows_remote=108474 dup_rows=6262 dup_rows_remote=6262 dedup_ratio=0.047775 router_tokens_per_expert=[3, 0, 3, 0, 8, 0, 2, 0, 8192, 8179, 8147, 2, 0, 3279, 0, 0, 97, 0, 8192, 7, 8168, 0, 0, 0, 0, 4867, 0, 8177, 8185, 21, 7, 0] payload_rows_per_dst=[16304, 16384, 16358, 10316, 16384, 16336, 16350, 16378] dup_rows_per_dst=[6, 16, 4, 32, 208, 0, 5948, 48] combine_rows_per_src=[16304, 16384, 16358, 10316, 16384, 16336, 16350, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=795017216 dispatch_meta_bytes=454728 combine_payload_bytes=795017216 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=795471944 ingress_bytes=1675821056 link_bytes=1675821056 token_bytes=1590489160 weight_movement_bytes=880803840 total_nvlink_bytes=2471293000 weight_to_token_ratio=0.553794 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.383898391e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120298 payload_rows_remote=103914 dup_rows=10774 dup_rows_remote=8618 dedup_ratio=0.082199 router_tokens_per_expert=[8187, 4, 8124, 2, 95, 1, 90, 8138, 6, 8089, 1, 6, 27, 51, 0, 1, 8156, 8186, 400, 0, 31, 7597, 2, 3, 1, 7670, 35, 523, 86, 2, 9, 13] payload_rows_per_dst=[16384, 16380, 16194, 15026, 16066, 16146, 8392, 15710] dup_rows_per_dst=[2156, 268, 378, 8, 728, 7184, 2, 50] combine_rows_per_src=[16384, 16380, 16194, 15026, 16066, 16146, 8392, 15710] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130842 payload_rows_remote=114480 dup_rows=230 dup_rows_remote=220 dedup_ratio=0.001755 router_tokens_per_expert=[8142, 0, 8186, 0, 0, 0, 0, 8191, 0, 8157, 0, 1, 64, 0, 35, 8, 8191, 8172, 0, 20, 0, 8173, 6, 0, 0, 8181, 0, 5, 3, 0, 0, 1] payload_rows_per_dst=[16284, 16382, 16316, 16382, 16384, 16356, 16362, 16376] dup_rows_per_dst=[0, 0, 0, 204, 8, 2, 10, 6] combine_rows_per_src=[16284, 16382, 16316, 16382, 16384, 16356, 16362, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121184 payload_rows_remote=114478 dup_rows=9888 dup_rows_remote=9888 dedup_ratio=0.075439 router_tokens_per_expert=[19, 0, 0, 10, 8191, 0, 25, 6, 82, 14, 8163, 8170, 0, 317, 0, 3, 3353, 0, 0, 0, 8174, 0, 24, 0, 8004, 0, 2, 4676, 8111, 1, 0, 8191] payload_rows_per_dst=[16364, 16384, 16384, 16382, 6706, 16384, 16356, 16224] dup_rows_per_dst=[20, 60, 148, 640, 0, 12, 9008, 0] combine_rows_per_src=[16364, 16384, 16384, 16382, 6706, 16384, 16356, 16224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127576 payload_rows_remote=111194 dup_rows=3496 dup_rows_remote=3494 dedup_ratio=0.026672 router_tokens_per_expert=[7912, 0, 8191, 0, 0, 0, 0, 8189, 1, 8191, 0, 0, 212, 0, 0, 0, 8178, 8162, 1256, 17, 14, 6870, 51, 0, 0, 8048, 0, 48, 0, 0, 196, 0] payload_rows_per_dst=[15824, 16378, 16382, 16384, 16384, 13802, 16098, 16324] dup_rows_per_dst=[0, 0, 2, 422, 2546, 68, 94, 364] combine_rows_per_src=[15824, 16378, 16382, 16384, 16384, 13802, 16098, 16324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=888619008 dispatch_meta_bytes=458944 combine_payload_bytes=888619008 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=889077952 ingress_bytes=1240940544 link_bytes=1240940544 token_bytes=1777696960 weight_movement_bytes=352321536 total_nvlink_bytes=2130018496 weight_to_token_ratio=0.198190 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.948749052e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121752 payload_rows_remote=105368 dup_rows=9320 dup_rows_remote=9284 dedup_ratio=0.071106 router_tokens_per_expert=[2, 0, 1, 0, 9, 1, 0, 9, 8189, 7961, 8180, 0, 0, 3170, 0, 1, 9, 0, 8183, 1622, 8133, 1, 24, 0, 0, 4618, 34, 7154, 8187, 46, 0, 2] payload_rows_per_dst=[16360, 16384, 15922, 14782, 16382, 16306, 9236, 16380] dup_rows_per_dst=[2, 36, 0, 5868, 3246, 10, 68, 90] combine_rows_per_src=[16360, 16384, 15922, 14782, 16382, 16306, 9236, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=851263488 dispatch_meta_bytes=450128 combine_payload_bytes=851263488 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=851713616 ingress_bytes=1732067328 link_bytes=1732067328 token_bytes=1702977104 weight_movement_bytes=880803840 total_nvlink_bytes=2583780944 weight_to_token_ratio=0.517214 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.698854036e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=937820160 dispatch_meta_bytes=458800 combine_payload_bytes=937820160 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=938278960 ingress_bytes=1466302464 link_bytes=1466302464 token_bytes=1876099120 weight_movement_bytes=528482304 total_nvlink_bytes=2404581424 weight_to_token_ratio=0.281692 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.210681734e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130930 payload_rows_remote=114556 dup_rows=142 dup_rows_remote=82 dedup_ratio=0.001083 router_tokens_per_expert=[8173, 2, 8178, 3, 4, 2, 1, 8191, 2, 8182, 10, 0, 35, 1, 2, 1, 8190, 7994, 194, 9, 0, 8185, 0, 3, 0, 8154, 2, 2, 16, 0, 0, 0] payload_rows_per_dst=[16354, 16384, 16384, 16374, 16364, 16374, 16312, 16384] dup_rows_per_dst=[2, 12, 4, 60, 30, 2, 4, 28] combine_rows_per_src=[16354, 16384, 16384, 16374, 16364, 16374, 16312, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=937803776 dispatch_meta_bytes=497464 combine_payload_bytes=937803776 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938301240 ingress_bytes=1290125312 link_bytes=1290125312 token_bytes=1876105016 weight_movement_bytes=352321536 total_nvlink_bytes=2228426552 weight_to_token_ratio=0.187794 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=27136 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224163222e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=910901248 dispatch_meta_bytes=458752 combine_payload_bytes=910901248 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=911360000 ingress_bytes=1263222784 link_bytes=1263222784 token_bytes=1822261248 weight_movement_bytes=352321536 total_nvlink_bytes=2174582784 weight_to_token_ratio=0.193343 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.073520295e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124950 payload_rows_remote=108566 dup_rows=6122 dup_rows_remote=5864 dedup_ratio=0.046707 router_tokens_per_expert=[7955, 0, 8087, 0, 0, 0, 5, 8140, 0, 8029, 17, 30, 24, 23, 7, 0, 8155, 8178, 342, 1, 0, 8164, 9, 0, 7, 8136, 20, 48, 47, 1, 0, 111] payload_rows_per_dst=[15294, 15968, 16118, 16220, 12240, 16346, 16380, 16384] dup_rows_per_dst=[0, 0, 300, 62, 666, 616, 4220, 258] combine_rows_per_src=[15294, 15968, 16118, 16220, 12240, 16346, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=863174656 dispatch_meta_bytes=458608 combine_payload_bytes=863174656 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=863633264 ingress_bytes=1391656960 link_bytes=1391656960 token_bytes=1726807920 weight_movement_bytes=528482304 total_nvlink_bytes=2255290224 weight_to_token_ratio=0.306046 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.015152 gemm_active_groups=13 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.792698070e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938442752 dispatch_meta_bytes=458552 combine_payload_bytes=938442752 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938901304 ingress_bytes=1290764288 link_bytes=1290764288 token_bytes=1877344056 weight_movement_bytes=352321536 total_nvlink_bytes=2229665592 weight_to_token_ratio=0.187670 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227741221e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124350 payload_rows_remote=107966 dup_rows=6722 dup_rows_remote=6710 dedup_ratio=0.051285 router_tokens_per_expert=[0, 0, 4874, 0, 8192, 0, 8182, 0, 5, 0, 8163, 5, 8186, 0, 2966, 4, 0, 0, 0, 8181, 0, 375, 7, 4, 0, 0, 6, 0, 8192, 2, 8192, 0] payload_rows_per_dst=[9748, 16364, 16342, 16382, 16362, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 5930, 0, 772, 12, 4] combine_rows_per_src=[9748, 16364, 16342, 16382, 16362, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130460 payload_rows_remote=114480 dup_rows=612 dup_rows_remote=584 dedup_ratio=0.004669 router_tokens_per_expert=[8125, 0, 8170, 0, 0, 0, 0, 8187, 1, 8160, 0, 0, 149, 0, 0, 1, 8190, 8191, 54, 29, 1, 7978, 25, 0, 0, 8192, 0, 78, 0, 0, 5, 0] payload_rows_per_dst=[16250, 16374, 16322, 16384, 16384, 15980, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 256, 166, 28, 156, 6] combine_rows_per_src=[16250, 16374, 16322, 16384, 16384, 15980, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115506 payload_rows_remote=99122 dup_rows=15566 dup_rows_remote=4828 dedup_ratio=0.118759 router_tokens_per_expert=[61, 23, 5358, 0, 8120, 1, 8000, 0, 73, 25, 8173, 139, 8119, 0, 3801, 7, 63, 0, 29, 7156, 49, 0, 0, 98, 1, 0, 0, 4, 8159, 0, 8073, 4] payload_rows_per_dst=[16384, 16118, 16384, 7616, 14438, 13068, 16306, 15192] dup_rows_per_dst=[10738, 3252, 1344, 0, 58, 98, 76, 0] combine_rows_per_src=[16384, 16118, 16384, 7616, 14438, 13068, 16306, 15192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=889372672 dispatch_meta_bytes=457720 combine_payload_bytes=889372672 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=889830392 ingress_bytes=1241694208 link_bytes=1241694208 token_bytes=1779203064 weight_movement_bytes=352321536 total_nvlink_bytes=2131524600 weight_to_token_ratio=0.198022 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.952969256e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122884 payload_rows_remote=106504 dup_rows=8188 dup_rows_remote=8182 dedup_ratio=0.062469 router_tokens_per_expert=[0, 0, 3974, 0, 8191, 17, 7996, 0, 0, 0, 8190, 3, 8177, 0, 4411, 0, 0, 0, 0, 8076, 11, 73, 36, 0, 0, 0, 0, 0, 8186, 2, 8192, 1] payload_rows_per_dst=[16368, 16022, 16380, 8822, 16152, 16382, 16376, 16382] dup_rows_per_dst=[7934, 4, 6, 0, 0, 240, 0, 4] combine_rows_per_src=[16368, 16022, 16380, 8822, 16152, 16382, 16376, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=884457472 dispatch_meta_bytes=458704 combine_payload_bytes=884457472 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=884916176 ingress_bytes=1412939776 link_bytes=1412939776 token_bytes=1769373648 weight_movement_bytes=528482304 total_nvlink_bytes=2297855952 weight_to_token_ratio=0.298683 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.911872956e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=937820160 dispatch_meta_bytes=460256 combine_payload_bytes=937820160 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938280416 ingress_bytes=1290141696 link_bytes=1290141696 token_bytes=1876100576 weight_movement_bytes=352321536 total_nvlink_bytes=2228422112 weight_to_token_ratio=0.187795 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224254966e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121026 payload_rows_remote=104676 dup_rows=10046 dup_rows_remote=9750 dedup_ratio=0.076645 router_tokens_per_expert=[6, 9, 11, 2, 108, 5, 19, 0, 8189, 8147, 8102, 0, 0, 4933, 0, 0, 0, 8, 8167, 148, 8076, 0, 0, 0, 0, 1179, 1866, 7846, 8173, 0, 542, 0] payload_rows_per_dst=[16384, 16204, 16294, 13052, 16350, 16152, 10208, 16382] dup_rows_per_dst=[56, 258, 0, 5006, 296, 0, 3382, 1048] combine_rows_per_src=[16384, 16204, 16294, 13052, 16350, 16152, 10208, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128800 payload_rows_remote=112416 dup_rows=2272 dup_rows_remote=2216 dedup_ratio=0.017334 router_tokens_per_expert=[8063, 13, 8185, 4, 27, 2, 0, 8191, 0, 8185, 0, 0, 285, 0, 0, 0, 8189, 8151, 164, 32, 72, 7866, 0, 0, 0, 8000, 0, 92, 9, 3, 3, 0] payload_rows_per_dst=[16142, 16384, 16370, 16370, 16366, 16054, 14730, 16384] dup_rows_per_dst=[18, 56, 0, 570, 322, 1114, 162, 30] combine_rows_per_src=[16142, 16384, 16370, 16370, 16366, 16054, 14730, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=812007424 dispatch_meta_bytes=415800 combine_payload_bytes=812007424 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=812423224 ingress_bytes=1164328960 link_bytes=1164328960 token_bytes=1624430648 weight_movement_bytes=352321536 total_nvlink_bytes=1976752184 weight_to_token_ratio=0.216889 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=20096 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.519756161e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120880 payload_rows_remote=109508 dup_rows=10192 dup_rows_remote=6430 dedup_ratio=0.077759 router_tokens_per_expert=[0, 0, 5844, 8, 8190, 50, 8186, 0, 2, 2, 7760, 24, 8089, 0, 2163, 0, 4, 0, 4, 8187, 0, 634, 0, 0, 0, 1, 4, 46, 8065, 85, 8188, 0] payload_rows_per_dst=[12170, 16372, 15570, 11372, 16374, 16380, 16384, 16258] dup_rows_per_dst=[3842, 100, 1068, 3762, 16, 1268, 94, 42] combine_rows_per_src=[12170, 16372, 15570, 11372, 16374, 16380, 16384, 16258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=872480768 dispatch_meta_bytes=458744 combine_payload_bytes=872480768 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=872939512 ingress_bytes=1400963072 link_bytes=1400963072 token_bytes=1745420280 weight_movement_bytes=528482304 total_nvlink_bytes=2273902584 weight_to_token_ratio=0.302782 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.844808412e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117038 payload_rows_remote=100692 dup_rows=14034 dup_rows_remote=13700 dedup_ratio=0.107071 router_tokens_per_expert=[0, 0, 5157, 2, 8176, 0, 7960, 19, 82, 17, 8125, 139, 8177, 0, 4703, 68, 0, 7, 0, 6420, 0, 23, 0, 81, 15, 24, 0, 0, 8176, 0, 8165, 0] payload_rows_per_dst=[10318, 13786, 14630, 16286, 13060, 16366, 16246, 16346] dup_rows_per_dst=[0, 2, 342, 6852, 6280, 194, 30, 334] combine_rows_per_src=[10318, 13786, 14630, 16286, 13060, 16366, 16246, 16346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=920911872 dispatch_meta_bytes=458528 combine_payload_bytes=920911872 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=921370400 ingress_bytes=1449394176 link_bytes=1449394176 token_bytes=1842282272 weight_movement_bytes=528482304 total_nvlink_bytes=2370764576 weight_to_token_ratio=0.286863 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.116002379e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=857505792 dispatch_meta_bytes=457704 combine_payload_bytes=857505792 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=857963496 ingress_bytes=1738309632 link_bytes=1738309632 token_bytes=1715469288 weight_movement_bytes=880803840 total_nvlink_bytes=2596273128 weight_to_token_ratio=0.513448 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.733808333e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=897089536 dispatch_meta_bytes=463752 combine_payload_bytes=897089536 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=897553288 ingress_bytes=1954054144 link_bytes=1954054144 token_bytes=1794642824 weight_movement_bytes=1056964608 total_nvlink_bytes=2851607432 weight_to_token_ratio=0.588955 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=18304 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.094188754e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 3, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126752 payload_rows_remote=110372 dup_rows=4320 dup_rows_remote=4320 dedup_ratio=0.032959 router_tokens_per_expert=[8191, 1, 0, 0, 3, 8192, 8192, 0, 0, 8190, 8192, 0, 0, 7663, 571, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8148, 0, 0, 0] payload_rows_per_dst=[16384, 12156, 16384, 16384, 16384, 16384, 16380, 16296] dup_rows_per_dst=[0, 6, 0, 4312, 0, 2, 0, 0] combine_rows_per_src=[16384, 12156, 16384, 16384, 16384, 16384, 16380, 16296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123824 payload_rows_remote=107440 dup_rows=7248 dup_rows_remote=7192 dedup_ratio=0.055298 router_tokens_per_expert=[2, 0, 4946, 0, 8145, 1, 8146, 0, 1, 0, 8188, 0, 8192, 3, 3578, 0, 0, 2, 1, 7864, 0, 3, 1, 71, 0, 0, 0, 1, 8190, 0, 8186, 15] payload_rows_per_dst=[9894, 16294, 16376, 16384, 15734, 16384, 16374, 16384] dup_rows_per_dst=[2, 0, 2, 7162, 0, 56, 0, 26] combine_rows_per_src=[9894, 16294, 16376, 16384, 15734, 16384, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=824868864 dispatch_meta_bytes=457568 combine_payload_bytes=824868864 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=825326432 ingress_bytes=1881833472 link_bytes=1881833472 token_bytes=1650195296 weight_movement_bytes=1056964608 total_nvlink_bytes=2707159904 weight_to_token_ratio=0.640509 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.053748193e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121106 payload_rows_remote=104764 dup_rows=9966 dup_rows_remote=9826 dedup_ratio=0.076035 router_tokens_per_expert=[8069, 25, 0, 95, 0, 8032, 8164, 0, 25, 8128, 7994, 17, 58, 7348, 0, 14, 0, 944, 8139, 22, 200, 4, 0, 0, 0, 15, 0, 12, 8158, 73, 0, 0] payload_rows_per_dst=[16342, 16312, 16290, 16248, 15616, 8030, 15900, 16368] dup_rows_per_dst=[140, 8692, 520, 450, 68, 2, 0, 94] combine_rows_per_src=[16342, 16312, 16290, 16248, 15616, 8030, 15900, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122872 payload_rows_remote=106490 dup_rows=8200 dup_rows_remote=8200 dedup_ratio=0.062561 router_tokens_per_expert=[8192, 0, 0, 6, 1, 8192, 8192, 0, 0, 8184, 8191, 0, 0, 4102, 0, 0, 0, 5, 8192, 0, 0, 0, 4034, 0, 53, 0, 0, 0, 8187, 5, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 8204, 16384, 16384, 16376, 16374] dup_rows_per_dst=[12, 2, 0, 0, 10, 8068, 98, 10] combine_rows_per_src=[16384, 16384, 16382, 8204, 16384, 16384, 16376, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=904167424 dispatch_meta_bytes=458768 combine_payload_bytes=904167424 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=904626192 ingress_bytes=1256488960 link_bytes=1256488960 token_bytes=1808793616 weight_movement_bytes=352321536 total_nvlink_bytes=2161115152 weight_to_token_ratio=0.194783 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.035813692e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=880148480 dispatch_meta_bytes=458528 combine_payload_bytes=880148480 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=880607008 ingress_bytes=1232470016 link_bytes=1232470016 token_bytes=1760755488 weight_movement_bytes=352321536 total_nvlink_bytes=2113077024 weight_to_token_ratio=0.200097 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.901317631e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121828 payload_rows_remote=105586 dup_rows=9244 dup_rows_remote=9240 dedup_ratio=0.070526 router_tokens_per_expert=[0, 5, 4160, 0, 8192, 0, 8120, 3, 0, 1, 8126, 0, 8186, 0, 3818, 0, 0, 9, 0, 8121, 0, 397, 3, 0, 0, 7, 0, 0, 8102, 101, 8185, 0] payload_rows_per_dst=[16382, 16242, 16254, 7636, 16260, 16384, 16380, 16290] dup_rows_per_dst=[8320, 4, 0, 0, 0, 800, 0, 120] combine_rows_per_src=[16382, 16242, 16254, 7636, 16260, 16384, 16380, 16290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=858226688 dispatch_meta_bytes=458360 combine_payload_bytes=858226688 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=858685048 ingress_bytes=1386708992 link_bytes=1386708992 token_bytes=1716911736 weight_movement_bytes=528482304 total_nvlink_bytes=2245394040 weight_to_token_ratio=0.307810 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.764991514e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124662 payload_rows_remote=111842 dup_rows=6410 dup_rows_remote=5322 dedup_ratio=0.048904 router_tokens_per_expert=[8185, 0, 0, 7, 8, 8186, 7419, 2, 2, 8188, 8189, 5, 0, 5958, 994, 2, 3, 0, 8181, 4, 2, 0, 2574, 6, 60, 0, 0, 10, 7541, 3, 7, 0] payload_rows_per_dst=[16382, 14854, 16382, 12820, 16368, 16374, 16384, 15098] dup_rows_per_dst=[2, 4, 10, 1088, 8, 5162, 132, 4] combine_rows_per_src=[16382, 14854, 16382, 12820, 16368, 16374, 16384, 15098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125534 payload_rows_remote=109150 dup_rows=5538 dup_rows_remote=5504 dedup_ratio=0.042252 router_tokens_per_expert=[8182, 0, 8165, 0, 65, 1, 0, 8177, 9, 8127, 15, 0, 0, 0, 0, 10, 8192, 8179, 30, 0, 0, 5500, 0, 0, 10, 8192, 0, 2681, 0, 1, 0, 0] payload_rows_per_dst=[16364, 16384, 16284, 16350, 16384, 11000, 16384, 16384] dup_rows_per_dst=[0, 102, 18, 0, 34, 0, 5382, 2] combine_rows_per_src=[16364, 16384, 16284, 16350, 16384, 11000, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=872366080 dispatch_meta_bytes=458760 combine_payload_bytes=872366080 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=872824840 ingress_bytes=1753169920 link_bytes=1753169920 token_bytes=1745190920 weight_movement_bytes=880803840 total_nvlink_bytes=2625994760 weight_to_token_ratio=0.504703 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.817019743e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=864960512 dispatch_meta_bytes=459304 combine_payload_bytes=864960512 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=865419816 ingress_bytes=1921925120 link_bytes=1921925120 token_bytes=1730380328 weight_movement_bytes=1056964608 total_nvlink_bytes=2787344936 weight_to_token_ratio=0.610828 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.076197842e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122212 payload_rows_remote=105832 dup_rows=8860 dup_rows_remote=7308 dedup_ratio=0.067596 router_tokens_per_expert=[8150, 0, 4, 122, 0, 8177, 8161, 36, 6, 8163, 8085, 8, 13, 8114, 0, 0, 8, 22, 8178, 12, 0, 0, 0, 0, 46, 0, 0, 3, 8159, 33, 36, 0] payload_rows_per_dst=[16298, 8228, 15946, 16288, 16384, 16346, 16342, 16380] dup_rows_per_dst=[168, 0, 18, 6734, 298, 0, 90, 1552] combine_rows_per_src=[16298, 8228, 15946, 16288, 16384, 16346, 16342, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=916209664 dispatch_meta_bytes=468656 combine_payload_bytes=916209664 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=916678320 ingress_bytes=1268531200 link_bytes=1268531200 token_bytes=1832887984 weight_movement_bytes=352321536 total_nvlink_bytes=2185209520 weight_to_token_ratio=0.192222 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.103245209e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=894156800 dispatch_meta_bytes=458616 combine_payload_bytes=894156800 prefetch_bytes=352321536 reduce_bytes=1233125376 egress_bytes=894615416 ingress_bytes=2479603712 link_bytes=2479603712 token_bytes=1788772216 weight_movement_bytes=1585446912 total_nvlink_bytes=3374219128 weight_to_token_ratio=0.886332 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.388474575e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130698 payload_rows_remote=114314 dup_rows=374 dup_rows_remote=354 dedup_ratio=0.002853 router_tokens_per_expert=[0, 8059, 0, 0, 8189, 0, 0, 0, 8192, 8189, 0, 0, 8192, 77, 8141, 1, 0, 0, 7, 0, 0, 3, 8189, 0, 3, 4, 4, 8191, 95, 0, 0, 0] payload_rows_per_dst=[16118, 16378, 16384, 16284, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 154, 14, 0, 20, 186] combine_rows_per_src=[16118, 16378, 16384, 16284, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=866975744 dispatch_meta_bytes=452560 combine_payload_bytes=866975744 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=867428304 ingress_bytes=1219297280 link_bytes=1219297280 token_bytes=1734404048 weight_movement_bytes=352321536 total_nvlink_bytes=2086725584 weight_to_token_ratio=0.203137 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.827555808e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130736 payload_rows_remote=114376 dup_rows=336 dup_rows_remote=54 dedup_ratio=0.002563 router_tokens_per_expert=[8192, 6, 0, 0, 0, 8180, 8191, 0, 0, 8161, 8187, 0, 0, 8048, 0, 0, 0, 1, 8190, 0, 0, 0, 141, 0, 28, 2, 0, 0, 8187, 5, 17, 0] payload_rows_per_dst=[16384, 16382, 16374, 16096, 16380, 16360, 16378, 16382] dup_rows_per_dst=[12, 0, 0, 0, 2, 282, 4, 36] combine_rows_per_src=[16384, 16382, 16374, 16096, 16380, 16360, 16378, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111244 payload_rows_remote=95040 dup_rows=19828 dup_rows_remote=11068 dedup_ratio=0.151276 router_tokens_per_expert=[0, 6229, 15, 20, 8106, 11, 6, 93, 8091, 7954, 0, 0, 8137, 0, 7830, 0, 29, 0, 0, 15, 62, 133, 8163, 0, 21, 155, 0, 8058, 2195, 11, 116, 86] payload_rows_per_dst=[16204, 16384, 15078, 8002, 16146, 13832, 14386, 11212] dup_rows_per_dst=[8760, 1178, 9288, 32, 30, 108, 0, 432] combine_rows_per_src=[16204, 16384, 15078, 8002, 16146, 13832, 14386, 11212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129938 payload_rows_remote=113620 dup_rows=1134 dup_rows_remote=1132 dedup_ratio=0.008652 router_tokens_per_expert=[0, 7756, 0, 0, 8158, 1, 0, 0, 8158, 8192, 0, 2, 8191, 315, 8127, 0, 1, 0, 6, 0, 0, 142, 8185, 97, 0, 0, 11, 8192, 2, 0, 0, 0] payload_rows_per_dst=[15512, 16318, 16318, 16254, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 630, 12, 464, 22, 4] combine_rows_per_src=[15512, 16318, 16318, 16254, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=936460288 dispatch_meta_bytes=458672 combine_payload_bytes=936460288 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=936918960 ingress_bytes=1641103360 link_bytes=1641103360 token_bytes=1873379248 weight_movement_bytes=704643072 total_nvlink_bytes=2578022320 weight_to_token_ratio=0.376135 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.189493786e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125502 payload_rows_remote=109146 dup_rows=5570 dup_rows_remote=260 dedup_ratio=0.042496 router_tokens_per_expert=[8189, 0, 0, 0, 75, 8192, 8173, 5, 2, 8191, 8188, 0, 2, 8073, 0, 0, 0, 18, 8191, 3, 15, 0, 14, 0, 1, 0, 93, 1, 8110, 0, 0, 0] payload_rows_per_dst=[16378, 16356, 16380, 16150, 16384, 11250, 16384, 16220] dup_rows_per_dst=[0, 5310, 0, 0, 40, 30, 190, 0] combine_rows_per_src=[16378, 16356, 16380, 16150, 16384, 11250, 16384, 16220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=936968192 dispatch_meta_bytes=457720 combine_payload_bytes=936968192 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937425912 ingress_bytes=1289289728 link_bytes=1289289728 token_bytes=1874394104 weight_movement_bytes=352321536 total_nvlink_bytes=2226715640 weight_to_token_ratio=0.187966 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.219484301e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122150 payload_rows_remote=105766 dup_rows=8922 dup_rows_remote=744 dedup_ratio=0.068069 router_tokens_per_expert=[5, 8032, 7, 20, 7942, 2, 0, 0, 4112, 8174, 43, 10, 8188, 4108, 8171, 2, 4, 1, 11, 0, 0, 0, 8182, 12, 1, 231, 44, 8179, 44, 0, 11, 0] payload_rows_per_dst=[16118, 15888, 8260, 16384, 16384, 16370, 16384, 16362] dup_rows_per_dst=[10, 0, 70, 8178, 24, 18, 526, 96] combine_rows_per_src=[16118, 15888, 8260, 16384, 16384, 16370, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=778567680 dispatch_meta_bytes=424432 combine_payload_bytes=778567680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=778992112 ingress_bytes=1130889216 link_bytes=1130889216 token_bytes=1557559792 weight_movement_bytes=352321536 total_nvlink_bytes=1909881328 weight_to_token_ratio=0.226201 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=30592 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.332507553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123452 payload_rows_remote=107138 dup_rows=7620 dup_rows_remote=7596 dedup_ratio=0.058136 router_tokens_per_expert=[0, 8, 4461, 17, 8119, 97, 8158, 1, 0, 0, 8184, 1, 8180, 12, 3242, 0, 9, 3, 0, 8157, 0, 4, 0, 25, 0, 4, 0, 26, 8163, 490, 8175, 0] payload_rows_per_dst=[8956, 16382, 16368, 16384, 16314, 16282, 16382, 16384] dup_rows_per_dst=[16, 130, 2, 6484, 24, 14, 28, 922] combine_rows_per_src=[8956, 16382, 16368, 16384, 16314, 16282, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27502237496 prefetch_bytes=5284823040 reduce_bytes=4051697664 weight_movement_bytes=9336520704 egress_bytes=13754718008 ingress_bytes=23084040192 link_bytes=23084040192 total_nvlink_bytes=36838758200 weight_to_token_ratio=0.339482 prefetch_share_of_ingress=0.228938 dedup_ratio=0.076545 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=23 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018821 gemm_inner_pad_upper_bound=23368 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.078811833e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 3, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=930775040 dispatch_meta_bytes=459008 combine_payload_bytes=930775040 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=931234048 ingress_bytes=1811578880 link_bytes=1811578880 token_bytes=1862009088 weight_movement_bytes=880803840 total_nvlink_bytes=2742812928 weight_to_token_ratio=0.473039 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.014408554e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=894124032 dispatch_meta_bytes=437624 combine_payload_bytes=894124032 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=894561656 ingress_bytes=2127249408 link_bytes=2127249408 token_bytes=1788685688 weight_movement_bytes=1233125376 total_nvlink_bytes=3021811064 weight_to_token_ratio=0.689403 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.191170873e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127334 payload_rows_remote=110950 dup_rows=3738 dup_rows_remote=2260 dedup_ratio=0.028519 router_tokens_per_expert=[41, 7212, 31, 0, 8175, 8, 0, 3, 8171, 8190, 122, 0, 8164, 0, 8001, 0, 17, 27, 9, 23, 21, 245, 8167, 0, 2, 2, 0, 8164, 734, 2, 0, 5] payload_rows_per_dst=[14562, 16366, 16342, 14592, 16378, 16378, 16332, 16384] dup_rows_per_dst=[6, 6, 244, 0, 108, 1606, 290, 1478] combine_rows_per_src=[14562, 16366, 16342, 14592, 16378, 16378, 16332, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=25893128696 prefetch_bytes=5460983808 reduce_bytes=7398752256 weight_movement_bytes=12859736064 egress_bytes=12950079992 ingress_bytes=25802784768 link_bytes=25802784768 total_nvlink_bytes=38752864760 weight_to_token_ratio=0.496647 prefetch_share_of_ingress=0.211643 dedup_ratio=0.122522 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=42 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018821 gemm_inner_pad_upper_bound=23622 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.030301506e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=866435072 dispatch_meta_bytes=426040 combine_payload_bytes=866435072 prefetch_bytes=176160768 reduce_bytes=1233125376 egress_bytes=866861112 ingress_bytes=2275721216 link_bytes=2275721216 token_bytes=1733296184 weight_movement_bytes=1409286144 total_nvlink_bytes=3142582328 weight_to_token_ratio=0.813067 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.274308888e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=877674496 dispatch_meta_bytes=458936 combine_payload_bytes=877674496 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=878133432 ingress_bytes=1229996032 link_bytes=1229996032 token_bytes=1755807928 weight_movement_bytes=352321536 total_nvlink_bytes=2108129464 weight_to_token_ratio=0.200661 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.887464354e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28320766456 prefetch_bytes=4756340736 reduce_bytes=5989466112 weight_movement_bytes=10745806848 egress_bytes=14164023800 ingress_bytes=24902549504 link_bytes=24902549504 total_nvlink_bytes=39066573304 weight_to_token_ratio=0.379432 prefetch_share_of_ingress=0.190998 dedup_ratio=0.053041 prefetch_slots=54 unused_prefetch_slots=202 slot_utilization=0.210938 reduce_slots=34 remote_demand=27 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020005 gemm_inner_pad_upper_bound=23114 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.715242649e-05 prefetch_bytes_per_gemm_flop=1.664595170e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=908902400 dispatch_meta_bytes=452840 combine_payload_bytes=908902400 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=909355240 ingress_bytes=1261223936 link_bytes=1261223936 token_bytes=1818257640 weight_movement_bytes=352321536 total_nvlink_bytes=2170579176 weight_to_token_ratio=0.193769 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.062327581e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125642 payload_rows_remote=109260 dup_rows=5430 dup_rows_remote=5346 dedup_ratio=0.041428 router_tokens_per_expert=[0, 5552, 0, 0, 8186, 0, 0, 0, 8191, 8189, 85, 0, 8180, 1440, 8033, 0, 20, 0, 0, 0, 5, 40, 8185, 3, 0, 0, 0, 8190, 1234, 0, 3, 0] payload_rows_per_dst=[11104, 16372, 16382, 16254, 16384, 16382, 16380, 16384] dup_rows_per_dst=[0, 0, 170, 2692, 16, 84, 0, 2468] combine_rows_per_src=[11104, 16372, 16382, 16254, 16384, 16382, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28541640152 prefetch_bytes=4756340736 reduce_bytes=5460983808 weight_movement_bytes=10217324544 egress_bytes=14274452952 ingress_bytes=24484511744 link_bytes=24484511744 total_nvlink_bytes=38758964696 weight_to_token_ratio=0.357980 prefetch_share_of_ingress=0.194259 dedup_ratio=0.050439 prefetch_slots=54 unused_prefetch_slots=202 slot_utilization=0.210938 reduce_slots=31 remote_demand=27 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018584 gemm_inner_pad_upper_bound=23241 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.568940339e-05 prefetch_bytes_per_gemm_flop=1.664595170e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125252 payload_rows_remote=108892 dup_rows=5820 dup_rows_remote=5820 dedup_ratio=0.044403 router_tokens_per_expert=[0, 7999, 0, 0, 8180, 0, 0, 0, 8064, 8179, 40, 0, 8191, 3, 8184, 0, 0, 1, 3, 3, 0, 39, 8190, 0, 0, 75, 193, 8190, 0, 2, 0, 0] payload_rows_per_dst=[16384, 16360, 16208, 10790, 16384, 16384, 16384, 16358] dup_rows_per_dst=[5198, 0, 0, 0, 12, 74, 532, 4] combine_rows_per_src=[16384, 16360, 16208, 10790, 16384, 16384, 16384, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=26059306616 prefetch_bytes=5108662272 reduce_bytes=4051697664 weight_movement_bytes=9160359936 egress_bytes=13033207416 ingress_bytes=22186459136 link_bytes=22186459136 total_nvlink_bytes=35219666552 weight_to_token_ratio=0.351520 prefetch_share_of_ingress=0.230260 dedup_ratio=0.118377 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=23 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019650 gemm_inner_pad_upper_bound=22860 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.764681880e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=895057920 dispatch_meta_bytes=458424 combine_payload_bytes=895057920 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=895516344 ingress_bytes=1599700992 link_bytes=1599700992 token_bytes=1790574264 weight_movement_bytes=704643072 total_nvlink_bytes=2495217336 weight_to_token_ratio=0.393529 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.957657807e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130484 payload_rows_remote=114118 dup_rows=588 dup_rows_remote=208 dedup_ratio=0.004486 router_tokens_per_expert=[8092, 0, 1, 0, 0, 8171, 8178, 15, 3, 8178, 8190, 0, 12, 8000, 131, 6, 0, 191, 8182, 0, 1, 0, 27, 0, 10, 0, 0, 19, 8093, 0, 26, 10] payload_rows_per_dst=[16186, 16384, 16384, 16188, 16366, 16344, 16384, 16248] dup_rows_per_dst=[0, 2, 2, 110, 380, 54, 30, 10] combine_rows_per_src=[16186, 16384, 16384, 16188, 16366, 16344, 16384, 16248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=892043264 dispatch_meta_bytes=458848 combine_payload_bytes=892043264 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=892502112 ingress_bytes=1244364800 link_bytes=1244364800 token_bytes=1784545376 weight_movement_bytes=352321536 total_nvlink_bytes=2136866912 weight_to_token_ratio=0.197429 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.967923457e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28641042032 prefetch_bytes=5460983808 reduce_bytes=3699376128 weight_movement_bytes=9160359936 egress_bytes=14324211312 ingress_bytes=23477190656 link_bytes=23477190656 total_nvlink_bytes=37801401968 weight_to_token_ratio=0.319833 prefetch_share_of_ingress=0.232608 dedup_ratio=0.049185 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=21 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018229 gemm_inner_pad_upper_bound=23876 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.216404238e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=934854656 dispatch_meta_bytes=457304 combine_payload_bytes=934854656 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=935311960 ingress_bytes=1815658496 link_bytes=1815658496 token_bytes=1870166616 weight_movement_bytes=880803840 total_nvlink_bytes=2750970456 weight_to_token_ratio=0.470976 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.016692968e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27921287392 prefetch_bytes=4932501504 reduce_bytes=4756340736 weight_movement_bytes=9688842240 egress_bytes=13964314848 ingress_bytes=23645814784 link_bytes=23645814784 total_nvlink_bytes=37610129632 weight_to_token_ratio=0.347006 prefetch_share_of_ingress=0.208599 dedup_ratio=0.069111 prefetch_slots=56 unused_prefetch_slots=200 slot_utilization=0.218750 reduce_slots=27 remote_demand=28 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018821 gemm_inner_pad_upper_bound=23241 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.275418284e-05 prefetch_bytes_per_gemm_flop=1.726246843e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124966 payload_rows_remote=108582 dup_rows=6106 dup_rows_remote=6078 dedup_ratio=0.046585 router_tokens_per_expert=[7, 5541, 0, 0, 7903, 8, 0, 0, 8163, 8090, 0, 0, 8191, 2221, 8192, 0, 0, 0, 12, 3, 0, 118, 8184, 0, 0, 148, 198, 8178, 366, 1, 2, 10] payload_rows_per_dst=[11092, 15822, 16326, 16384, 16384, 16368, 16384, 16206] dup_rows_per_dst=[4, 0, 0, 4442, 28, 236, 664, 732] combine_rows_per_src=[11092, 15822, 16326, 16384, 16384, 16368, 16384, 16206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 6, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=889503744 dispatch_meta_bytes=458640 combine_payload_bytes=889503744 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=889962384 ingress_bytes=1417986048 link_bytes=1417986048 token_bytes=1779466128 weight_movement_bytes=528482304 total_nvlink_bytes=2307948432 weight_to_token_ratio=0.296989 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.015152 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.940129972e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28846433832 prefetch_bytes=5284823040 reduce_bytes=5637144576 weight_movement_bytes=10921967616 egress_bytes=14426924584 ingress_bytes=25341476864 link_bytes=25341476864 total_nvlink_bytes=39768401448 weight_to_token_ratio=0.378625 prefetch_share_of_ingress=0.208544 dedup_ratio=0.049506 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=32 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018821 gemm_inner_pad_upper_bound=23495 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.868855774e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none - [2026-09-16 21:37:05] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 569.3 | throughput per GPU (TFLOP/s/GPU): 585.6 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.322889E+01 | loss scale: 1.0 | grad norm: 31.680 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=91166 payload_rows_remote=74952 dup_rows=39906 dup_rows_remote=27150 dedup_ratio=0.304459 router_tokens_per_expert=[13, 7946, 4096, 9, 5149, 0, 1739, 10, 5881, 411, 3148, 3425, 1981, 1091, 17, 1123, 1492, 3186, 2560, 3017, 4117, 2171, 3931, 49, 1, 1512, 449, 1233, 2221, 76, 3229, 253] payload_rows_per_dst=[16214, 14082, 14770, 7184, 9968, 13060, 6216, 9672] dup_rows_per_dst=[12756, 6420, 7834, 1240, 2852, 6108, 810, 1886] combine_rows_per_src=[16214, 14082, 14770, 7184, 9968, 13060, 6216, 9672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98310 payload_rows_remote=84944 dup_rows=32762 dup_rows_remote=25706 dedup_ratio=0.249954 router_tokens_per_expert=[13, 7994, 4457, 10, 5200, 2, 1647, 39, 5935, 362, 2688, 3890, 2271, 459, 59, 893, 1894, 2710, 2821, 2384, 4420, 2012, 4090, 83, 0, 1537, 490, 1385, 2069, 107, 3222, 393] payload_rows_per_dst=[8944, 11572, 10792, 16230, 12018, 11970, 13418, 13366] dup_rows_per_dst=[16, 2204, 3088, 7122, 5038, 2962, 5276, 7056] combine_rows_per_src=[8944, 11572, 10792, 16230, 12018, 11970, 13418, 13366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95758 payload_rows_remote=84866 dup_rows=35314 dup_rows_remote=32026 dedup_ratio=0.269424 router_tokens_per_expert=[21, 8063, 4292, 12, 4999, 5, 2087, 9, 5985, 481, 3536, 3390, 1909, 884, 20, 910, 1799, 3226, 2363, 2674, 4983, 2102, 3196, 29, 0, 1895, 348, 1036, 1730, 68, 3326, 158] payload_rows_per_dst=[12748, 11726, 11422, 10892, 12640, 10848, 13384, 12098] dup_rows_per_dst=[5294, 2474, 4136, 3288, 6740, 3798, 5144, 4440] combine_rows_per_src=[12748, 11726, 11422, 10892, 12640, 10848, 13384, 12098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=93422 payload_rows_remote=80070 dup_rows=37650 dup_rows_remote=33794 dedup_ratio=0.287247 router_tokens_per_expert=[21, 7658, 3294, 125, 5813, 2, 1739, 41, 5693, 769, 3424, 3008, 1622, 873, 72, 1179, 1723, 2450, 1966, 2883, 5114, 2772, 3857, 87, 3, 1788, 374, 1958, 1621, 47, 3347, 213] payload_rows_per_dst=[15872, 13352, 13340, 6504, 9586, 14370, 11104, 9294] dup_rows_per_dst=[9154, 3856, 5666, 988, 3612, 9290, 3922, 1162] combine_rows_per_src=[15872, 13352, 13340, 6504, 9586, 14370, 11104, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98124 payload_rows_remote=86812 dup_rows=32948 dup_rows_remote=30410 dedup_ratio=0.251373 router_tokens_per_expert=[17, 8074, 4544, 18, 6093, 11, 2384, 28, 5990, 199, 2682, 3526, 2002, 714, 9, 1230, 1547, 3633, 1645, 2775, 4303, 1755, 3936, 68, 2, 1744, 360, 1523, 1470, 47, 3030, 177] payload_rows_per_dst=[9148, 13324, 10296, 16296, 11718, 11312, 13614, 12416] dup_rows_per_dst=[10, 3708, 2518, 7762, 5150, 2538, 5624, 5638] combine_rows_per_src=[9148, 13324, 10296, 16296, 11718, 11312, 13614, 12416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95424 payload_rows_remote=83542 dup_rows=35648 dup_rows_remote=30946 dedup_ratio=0.271973 router_tokens_per_expert=[14, 7937, 4047, 12, 5038, 2, 2026, 19, 6110, 326, 3698, 3684, 1700, 879, 49, 982, 1537, 2908, 1848, 3195, 5031, 1758, 3504, 56, 0, 1534, 363, 827, 2605, 125, 3594, 128] payload_rows_per_dst=[16240, 12040, 11882, 6482, 11398, 12674, 13326, 11382] dup_rows_per_dst=[8794, 3114, 4702, 738, 4412, 5756, 4342, 3790] combine_rows_per_src=[16240, 12040, 11882, 6482, 11398, 12674, 13326, 11382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=614006784 dispatch_meta_bytes=408408 combine_payload_bytes=614006784 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=614415192 ingress_bytes=1318649856 link_bytes=1318649856 token_bytes=1228421976 weight_movement_bytes=704643072 total_nvlink_bytes=1933065048 weight_to_token_ratio=0.573616 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=16768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.383888762e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=695861248 dispatch_meta_bytes=442600 combine_payload_bytes=695861248 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=696303848 ingress_bytes=1048182784 link_bytes=1048182784 token_bytes=1392165096 weight_movement_bytes=352321536 total_nvlink_bytes=1744486632 weight_to_token_ratio=0.253075 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=13568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.869386057e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=695222272 dispatch_meta_bytes=467568 combine_payload_bytes=695222272 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=695689840 ingress_bytes=1047543808 link_bytes=1047543808 token_bytes=1390912112 weight_movement_bytes=352321536 total_nvlink_bytes=1743233648 weight_to_token_ratio=0.253303 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=18304 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.865808058e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=655933440 dispatch_meta_bytes=455456 combine_payload_bytes=655933440 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=656388896 ingress_bytes=1008254976 link_bytes=1008254976 token_bytes=1312322336 weight_movement_bytes=352321536 total_nvlink_bytes=1664643872 weight_to_token_ratio=0.268472 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=23168 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.645806999e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=711163904 dispatch_meta_bytes=468888 combine_payload_bytes=711163904 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=711632792 ingress_bytes=1415806976 link_bytes=1415806976 token_bytes=1422796696 weight_movement_bytes=704643072 total_nvlink_bytes=2127439768 weight_to_token_ratio=0.495252 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=14976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.927928079e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=684376064 dispatch_meta_bytes=457952 combine_payload_bytes=684376064 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=684834016 ingress_bytes=1565179904 link_bytes=1565179904 token_bytes=1369210080 weight_movement_bytes=880803840 total_nvlink_bytes=2250013920 weight_to_token_ratio=0.643293 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=14208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.764354124e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98128 payload_rows_remote=84224 dup_rows=32944 dup_rows_remote=28332 dedup_ratio=0.251343 router_tokens_per_expert=[9, 8094, 3844, 8, 5689, 0, 2480, 7, 6439, 186, 3940, 4163, 1672, 1072, 21, 1037, 1655, 3227, 1439, 2472, 4853, 2001, 3794, 14, 1, 1833, 344, 641, 1345, 47, 3093, 116] payload_rows_per_dst=[7704, 13016, 12300, 16268, 9970, 12026, 13904, 12940] dup_rows_per_dst=[18, 3336, 4278, 7524, 4416, 2792, 4612, 5968] combine_rows_per_src=[7704, 13016, 12300, 16268, 9970, 12026, 13904, 12940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98122 payload_rows_remote=84668 dup_rows=32950 dup_rows_remote=24850 dedup_ratio=0.251389 router_tokens_per_expert=[7, 7965, 4072, 1, 6284, 2, 2132, 6, 5273, 318, 2845, 3861, 1626, 887, 26, 773, 1661, 3769, 1840, 3507, 4152, 2248, 3476, 60, 1, 1500, 380, 1034, 1928, 51, 3569, 282] payload_rows_per_dst=[8154, 13444, 10956, 16098, 13454, 9910, 12790, 13316] dup_rows_per_dst=[6, 3404, 3092, 6456, 8100, 1658, 3586, 6648] combine_rows_per_src=[8154, 13444, 10956, 16098, 13454, 9910, 12790, 13316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96412 payload_rows_remote=80042 dup_rows=34660 dup_rows_remote=21166 dedup_ratio=0.264435 router_tokens_per_expert=[7, 1, 773, 8058, 0, 0, 8171, 8172, 0, 3546, 0, 0, 0, 0, 8189, 1, 1, 2, 8179, 0, 0, 1, 7597, 2387, 0, 25, 3551, 2, 6873, 0, 0, 0] payload_rows_per_dst=[1550, 0, 13018, 16380, 16360, 16382, 16352, 16370] dup_rows_per_dst=[12, 0, 2244, 0, 4, 12814, 6092, 13494] combine_rows_per_src=[1550, 0, 13018, 16380, 16360, 16382, 16352, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=103178 payload_rows_remote=86794 dup_rows=27894 dup_rows_remote=17378 dedup_ratio=0.212814 router_tokens_per_expert=[1, 0, 805, 8184, 0, 0, 8191, 8189, 1, 2877, 2, 0, 0, 0, 8188, 2, 3, 3, 8186, 0, 1, 0, 8186, 3472, 0, 33, 2751, 0, 6459, 0, 0, 2] payload_rows_per_dst=[16384, 16380, 9166, 16336, 16372, 10056, 5562, 12922] dup_rows_per_dst=[10516, 7504, 4786, 0, 12, 5070, 6, 0] combine_rows_per_src=[16384, 16380, 9166, 16336, 16372, 10056, 5562, 12922] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=689963008 dispatch_meta_bytes=450224 combine_payload_bytes=689963008 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=690413232 ingress_bytes=1042284544 link_bytes=1042284544 token_bytes=1380376240 weight_movement_bytes=352321536 total_nvlink_bytes=1732697776 weight_to_token_ratio=0.255236 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=20096 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.836358375e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111608 payload_rows_remote=99710 dup_rows=19464 dup_rows_remote=16814 dedup_ratio=0.148499 router_tokens_per_expert=[1, 18, 1407, 8024, 7, 6, 8096, 8148, 2, 3368, 1, 2, 7, 4, 8190, 0, 39, 1, 8087, 1, 6, 0, 8167, 3184, 1, 45, 4364, 21, 4319, 0, 18, 2] payload_rows_per_dst=[16152, 12850, 11676, 16382, 16242, 11898, 16384, 10024] dup_rows_per_dst=[2748, 10, 3174, 20, 148, 2650, 8836, 1878] combine_rows_per_src=[16152, 12850, 11676, 16382, 16242, 11898, 16384, 10024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105468 payload_rows_remote=89084 dup_rows=25604 dup_rows_remote=9346 dedup_ratio=0.195343 router_tokens_per_expert=[16, 2, 625, 8160, 13, 0, 8120, 8188, 0, 3478, 0, 4, 0, 0, 8143, 0, 40, 6, 8174, 0, 1, 0, 8163, 2084, 0, 17, 4449, 3, 5836, 0, 14, 0] payload_rows_per_dst=[16346, 16384, 10546, 16286, 16378, 8894, 8938, 11696] dup_rows_per_dst=[1260, 16258, 4554, 0, 62, 3466, 0, 4] combine_rows_per_src=[16346, 16384, 10546, 16286, 16378, 8894, 8938, 11696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=107570 payload_rows_remote=94740 dup_rows=23502 dup_rows_remote=19222 dedup_ratio=0.179306 router_tokens_per_expert=[28, 1, 2389, 8124, 7, 0, 8161, 8160, 1, 2726, 2, 0, 3, 37, 8186, 1, 0, 0, 8167, 12, 2, 0, 8078, 3320, 1, 78, 1337, 0, 6709, 1, 5, 0] payload_rows_per_dst=[16366, 16372, 12830, 16384, 16356, 12414, 3418, 13430] dup_rows_per_dst=[4718, 11976, 4280, 70, 2, 2210, 246, 0] combine_rows_per_src=[16366, 16372, 12830, 16384, 16356, 12414, 3418, 13430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 1, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117194 payload_rows_remote=100812 dup_rows=13878 dup_rows_remote=13856 dedup_ratio=0.105881 router_tokens_per_expert=[11, 11, 1248, 8186, 2, 2, 8178, 8150, 3, 4097, 0, 0, 1, 11, 8190, 0, 1, 0, 8159, 0, 1, 0, 8191, 2876, 0, 5, 1549, 0, 6628, 0, 36, 0] payload_rows_per_dst=[16378, 16360, 16312, 16382, 16320, 5754, 16378, 13310] dup_rows_per_dst=[2534, 4, 8194, 22, 0, 0, 3106, 18] combine_rows_per_src=[16378, 16360, 16312, 16382, 16320, 5754, 16378, 13310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 1, 2, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=655704064 dispatch_meta_bytes=404832 combine_payload_bytes=655704064 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=656108896 ingress_bytes=1008025600 link_bytes=1008025600 token_bytes=1311812960 weight_movement_bytes=352321536 total_nvlink_bytes=1664134496 weight_to_token_ratio=0.268576 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.644522589e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=711016448 dispatch_meta_bytes=416688 combine_payload_bytes=711016448 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=711433136 ingress_bytes=1415659520 link_bytes=1415659520 token_bytes=1422449584 weight_movement_bytes=704643072 total_nvlink_bytes=2127092656 weight_to_token_ratio=0.495373 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=27264 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.927102387e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=816824320 dispatch_meta_bytes=466096 combine_payload_bytes=816824320 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=817290416 ingress_bytes=1521467392 link_bytes=1521467392 token_bytes=1634114736 weight_movement_bytes=704643072 total_nvlink_bytes=2338757808 weight_to_token_ratio=0.431208 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=14592 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.519582303e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=729776128 dispatch_meta_bytes=393720 combine_payload_bytes=729776128 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=730169848 ingress_bytes=2491383808 link_bytes=2491383808 token_bytes=1459945976 weight_movement_bytes=1761607680 total_nvlink_bytes=3221553656 weight_to_token_ratio=1.206625 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=22272 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.395070937e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=776110080 dispatch_meta_bytes=455848 combine_payload_bytes=776110080 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=776565928 ingress_bytes=1128431616 link_bytes=1128431616 token_bytes=1552676008 weight_movement_bytes=352321536 total_nvlink_bytes=1904997544 weight_to_token_ratio=0.226912 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=22272 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.318746019e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=693600256 dispatch_meta_bytes=438072 combine_payload_bytes=693600256 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=694038328 ingress_bytes=1926725632 link_bytes=1926725632 token_bytes=1387638584 weight_movement_bytes=1233125376 total_nvlink_bytes=2620763960 weight_to_token_ratio=0.888650 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=9344 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.078885928e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=825851904 dispatch_meta_bytes=458672 combine_payload_bytes=825851904 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=826310576 ingress_bytes=1178173440 link_bytes=1178173440 token_bytes=1652162480 weight_movement_bytes=352321536 total_nvlink_bytes=2004484016 weight_to_token_ratio=0.213249 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.597279470e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98278 payload_rows_remote=81900 dup_rows=32794 dup_rows_remote=25430 dedup_ratio=0.250198 router_tokens_per_expert=[3, 0, 489, 8179, 4, 0, 8119, 8191, 15, 3052, 3, 1, 0, 5, 8185, 0, 0, 0, 8094, 0, 0, 1, 8173, 3928, 1, 3, 3681, 3, 5400, 0, 6, 0] payload_rows_per_dst=[8142, 624, 9164, 16378, 16188, 15798, 16378, 15606] dup_rows_per_dst=[308, 0, 5170, 54, 330, 10022, 7364, 9546] combine_rows_per_src=[8142, 624, 9164, 16378, 16188, 15798, 16378, 15606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127460 payload_rows_remote=111076 dup_rows=3612 dup_rows_remote=310 dedup_ratio=0.027557 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 8192, 1, 0, 0, 8190, 2, 7581, 8192, 2, 0, 0, 1, 8165, 12, 1, 0, 0, 7203, 0, 8177, 0, 8190, 2, 1622, 2] payload_rows_per_dst=[16290, 16108, 16382, 15166, 16332, 16384, 14414, 16384] dup_rows_per_dst=[0, 2, 222, 4, 0, 26, 56, 3302] combine_rows_per_src=[16290, 16108, 16382, 15166, 16332, 16384, 14414, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116262 payload_rows_remote=99906 dup_rows=14810 dup_rows_remote=14734 dedup_ratio=0.112991 router_tokens_per_expert=[4, 0, 2, 2, 8, 0, 4, 8186, 12, 0, 5, 8183, 0, 6972, 8170, 0, 0, 2, 0, 8185, 5, 31, 2, 5, 7407, 1, 8169, 0, 8186, 0, 1995, 0] payload_rows_per_dst=[16356, 16384, 16370, 16330, 16374, 15608, 14850, 3990] dup_rows_per_dst=[76, 58, 30, 12876, 342, 22, 1406, 0] combine_rows_per_src=[16356, 16384, 16370, 16330, 16374, 15608, 14850, 3990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129232 payload_rows_remote=112848 dup_rows=1840 dup_rows_remote=1666 dedup_ratio=0.014038 router_tokens_per_expert=[39, 7, 2, 4, 0, 16, 15, 8178, 0, 2, 14, 8121, 1, 7550, 8169, 6, 16, 3, 0, 7994, 75, 20, 0, 15, 8160, 1, 8172, 3, 8187, 1, 765, 0] payload_rows_per_dst=[16362, 16384, 16272, 15102, 16026, 16384, 16320, 16382] dup_rows_per_dst=[86, 34, 2, 12, 0, 174, 8, 1524] combine_rows_per_src=[16362, 16384, 16272, 15102, 16026, 16384, 16320, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122206 payload_rows_remote=105832 dup_rows=8866 dup_rows_remote=8842 dedup_ratio=0.067642 router_tokens_per_expert=[0, 10, 1, 2, 0, 2, 16, 8189, 1, 3, 20, 8175, 18, 6055, 8189, 5, 5, 3, 0, 8139, 54, 262, 0, 0, 7930, 1, 8171, 0, 8172, 0, 2106, 7] payload_rows_per_dst=[16360, 16384, 16374, 12132, 16286, 16378, 16102, 12190] dup_rows_per_dst=[8, 30, 24, 24, 8, 632, 7870, 270] combine_rows_per_src=[16360, 16384, 16374, 12132, 16286, 16378, 16102, 12190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120316 payload_rows_remote=103932 dup_rows=10756 dup_rows_remote=10622 dedup_ratio=0.082062 router_tokens_per_expert=[21, 0, 1, 0, 68, 0, 0, 8191, 0, 1, 0, 8155, 0, 7099, 8170, 331, 48, 0, 16, 7826, 0, 6, 1, 1, 5410, 0, 8186, 1, 8173, 3, 3828, 0] payload_rows_per_dst=[16374, 16384, 16312, 15436, 15746, 16342, 12984, 10738] dup_rows_per_dst=[42, 134, 0, 5902, 896, 14, 2940, 828] combine_rows_per_src=[16374, 16384, 16312, 15436, 15746, 16342, 12984, 10738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125730 payload_rows_remote=111802 dup_rows=5342 dup_rows_remote=5306 dedup_ratio=0.040756 router_tokens_per_expert=[1, 0, 0, 2, 3, 0, 5, 8192, 5, 7, 2, 8181, 0, 6949, 8184, 33, 0, 0, 0, 8181, 0, 2, 2, 0, 6783, 0, 8187, 2, 8170, 0, 2645, 0] payload_rows_per_dst=[16378, 16384, 16370, 13928, 16362, 16372, 13566, 16370] dup_rows_per_dst=[2, 16, 20, 36, 0, 4, 4, 5260] combine_rows_per_src=[16378, 16384, 16370, 13928, 16362, 16372, 13566, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 6, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=670924800 dispatch_meta_bytes=429320 combine_payload_bytes=670924800 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=671354120 ingress_bytes=1023246336 link_bytes=1023246336 token_bytes=1342278920 weight_movement_bytes=352321536 total_nvlink_bytes=1694600456 weight_to_token_ratio=0.262480 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=20864 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.729752358e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=909934592 dispatch_meta_bytes=445544 combine_payload_bytes=909934592 prefetch_bytes=176160768 reduce_bytes=1233125376 egress_bytes=910380136 ingress_bytes=2319220736 link_bytes=2319220736 token_bytes=1820314728 weight_movement_bytes=1409286144 total_nvlink_bytes=3229600872 weight_to_token_ratio=0.774199 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=26624 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.298666803e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=818429952 dispatch_meta_bytes=458560 combine_payload_bytes=818429952 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=818888512 ingress_bytes=1346912256 link_bytes=1346912256 token_bytes=1637318464 weight_movement_bytes=528482304 total_nvlink_bytes=2165800768 weight_to_token_ratio=0.322773 gemm_rows_defined=66688 gemm_rows_charged=67584 gemm_tail_pad_rows=896 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.013258 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.542146405e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119642 payload_rows_remote=103344 dup_rows=11430 dup_rows_remote=11430 dedup_ratio=0.087204 router_tokens_per_expert=[6, 0, 676, 8147, 0, 0, 8175, 8189, 16, 2164, 0, 0, 0, 0, 8188, 1, 0, 0, 8149, 0, 1, 0, 8139, 4467, 2, 0, 2824, 5, 6382, 0, 5, 0] payload_rows_per_dst=[16326, 16350, 13744, 16376, 16298, 11394, 16384, 12770] dup_rows_per_dst=[1332, 0, 2214, 2, 0, 2250, 5628, 4] combine_rows_per_src=[16326, 16350, 13744, 16376, 16298, 11394, 16384, 12770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=866975744 dispatch_meta_bytes=458696 combine_payload_bytes=866975744 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=867434440 ingress_bytes=1219297280 link_bytes=1219297280 token_bytes=1734410184 weight_movement_bytes=352321536 total_nvlink_bytes=2086731720 weight_to_token_ratio=0.203136 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.827555808e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=924450816 dispatch_meta_bytes=458056 combine_payload_bytes=924450816 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=924908872 ingress_bytes=1276772352 link_bytes=1276772352 token_bytes=1849359688 weight_movement_bytes=352321536 total_nvlink_bytes=2201681224 weight_to_token_ratio=0.190510 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.149392220e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=851410944 dispatch_meta_bytes=458216 combine_payload_bytes=851410944 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=851869160 ingress_bytes=1556054016 link_bytes=1556054016 token_bytes=1703280104 weight_movement_bytes=704643072 total_nvlink_bytes=2407923176 weight_to_token_ratio=0.413698 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713252961e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=915881984 dispatch_meta_bytes=468432 combine_payload_bytes=915881984 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=916350416 ingress_bytes=1620525056 link_bytes=1620525056 token_bytes=1832232400 weight_movement_bytes=704643072 total_nvlink_bytes=2536875472 weight_to_token_ratio=0.384582 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.074263873e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=846594048 dispatch_meta_bytes=459096 combine_payload_bytes=846594048 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=847053144 ingress_bytes=1198915584 link_bytes=1198915584 token_bytes=1693647192 weight_movement_bytes=352321536 total_nvlink_bytes=2045968728 weight_to_token_ratio=0.208025 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.713426818e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117330 payload_rows_remote=100946 dup_rows=13742 dup_rows_remote=13680 dedup_ratio=0.104843 router_tokens_per_expert=[8190, 0, 2, 0, 1, 1, 147, 8190, 1381, 0, 1, 375, 8192, 0, 1, 6415, 0, 8192, 8189, 0, 8064, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16380, 15710, 3670, 16288, 16358, 16156, 16384, 16384] dup_rows_per_dst=[2, 272, 228, 6766, 116, 6296, 0, 62] combine_rows_per_src=[16380, 15710, 3670, 16288, 16358, 16156, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126590 payload_rows_remote=111426 dup_rows=4482 dup_rows_remote=4482 dedup_ratio=0.034195 router_tokens_per_expert=[5, 2, 0, 0, 0, 0, 0, 8192, 0, 2, 4, 8148, 0, 6649, 8144, 1, 0, 0, 1, 8190, 24, 6, 0, 0, 7581, 1, 8168, 0, 8189, 6, 2222, 1] payload_rows_per_dst=[16344, 16384, 16304, 13298, 16382, 16330, 15164, 16384] dup_rows_per_dst=[6, 0, 4, 2, 0, 18, 0, 4452] combine_rows_per_src=[16344, 16384, 16304, 13298, 16382, 16330, 15164, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128658 payload_rows_remote=112274 dup_rows=2414 dup_rows_remote=2344 dedup_ratio=0.018417 router_tokens_per_expert=[8187, 0, 1, 6, 1, 2, 2, 8186, 92, 0, 2, 792, 8183, 0, 0, 7464, 0, 8189, 8189, 3, 8022, 0, 6, 2, 3, 12, 3, 4, 0, 8185, 0, 0] payload_rows_per_dst=[16384, 16376, 16376, 14958, 16246, 16056, 15958, 16304] dup_rows_per_dst=[70, 92, 1762, 408, 62, 4, 16, 0] combine_rows_per_src=[16384, 16376, 16376, 14958, 16246, 16056, 15958, 16304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129256 payload_rows_remote=112880 dup_rows=1816 dup_rows_remote=348 dedup_ratio=0.013855 router_tokens_per_expert=[8152, 0, 48, 6, 0, 0, 12, 8183, 409, 0, 0, 327, 8186, 0, 1, 7423, 9, 8178, 8168, 5, 8086, 4, 0, 1, 27, 97, 0, 0, 0, 8192, 22, 0] payload_rows_per_dst=[16376, 16368, 16376, 14848, 16348, 16172, 16384, 16384] dup_rows_per_dst=[36, 22, 1468, 0, 10, 10, 226, 44] combine_rows_per_src=[16376, 16368, 16376, 14848, 16348, 16172, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[3, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129836 payload_rows_remote=114066 dup_rows=1236 dup_rows_remote=1236 dedup_ratio=0.009430 router_tokens_per_expert=[8166, 10, 0, 0, 13, 4, 2, 8183, 213, 3, 7, 364, 8169, 0, 19, 7913, 2, 8178, 8188, 22, 7877, 0, 8, 0, 10, 0, 4, 0, 0, 8169, 12, 0] payload_rows_per_dst=[16346, 16384, 16378, 15864, 16384, 15770, 16370, 16340] dup_rows_per_dst=[6, 20, 1134, 0, 34, 0, 20, 22] combine_rows_per_src=[16346, 16384, 16378, 15864, 16384, 15770, 16370, 16340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130430 payload_rows_remote=114046 dup_rows=642 dup_rows_remote=536 dedup_ratio=0.004898 router_tokens_per_expert=[8149, 0, 1, 0, 1, 0, 54, 8190, 90, 0, 1, 139, 8109, 1, 0, 8068, 0, 8192, 8191, 0, 8084, 0, 0, 74, 0, 1, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16300, 16384, 16292, 16138, 16384, 16168, 16382, 16382] dup_rows_per_dst=[0, 106, 386, 0, 0, 148, 2, 0] combine_rows_per_src=[16300, 16384, 16292, 16138, 16384, 16168, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=826949632 dispatch_meta_bytes=458504 combine_payload_bytes=826949632 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=827408136 ingress_bytes=1355431936 link_bytes=1355431936 token_bytes=1654357768 weight_movement_bytes=528482304 total_nvlink_bytes=2182840072 weight_to_token_ratio=0.319449 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.589853056e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126360 payload_rows_remote=113012 dup_rows=4712 dup_rows_remote=4712 dedup_ratio=0.035950 router_tokens_per_expert=[8187, 0, 19, 0, 0, 0, 27, 8190, 1170, 0, 0, 1082, 8178, 0, 0, 6674, 2, 8180, 8192, 0, 7370, 2, 0, 7, 0, 63, 1, 0, 0, 8190, 0, 2] payload_rows_per_dst=[16380, 16380, 16378, 13348, 16366, 14742, 16382, 16384] dup_rows_per_dst=[32, 54, 4482, 0, 0, 16, 128, 0] combine_rows_per_src=[16380, 16380, 16378, 13348, 16366, 14742, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=912801792 dispatch_meta_bytes=463632 combine_payload_bytes=912801792 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=913265424 ingress_bytes=1793605632 link_bytes=1793605632 token_bytes=1826067216 weight_movement_bytes=880803840 total_nvlink_bytes=2706871056 weight_to_token_ratio=0.482350 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=29952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.004344285e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=919748608 dispatch_meta_bytes=458472 combine_payload_bytes=919748608 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=920207080 ingress_bytes=1272070144 link_bytes=1272070144 token_bytes=1839955688 weight_movement_bytes=352321536 total_nvlink_bytes=2192277224 weight_to_token_ratio=0.191484 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.123061818e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=924712960 dispatch_meta_bytes=452912 combine_payload_bytes=924712960 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=925165872 ingress_bytes=1277034496 link_bytes=1277034496 token_bytes=1849878832 weight_movement_bytes=352321536 total_nvlink_bytes=2202200368 weight_to_token_ratio=0.190457 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=28800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.150860117e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124414 payload_rows_remote=108096 dup_rows=6658 dup_rows_remote=6658 dedup_ratio=0.050797 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 1, 8192, 6, 6, 5, 8176, 0, 5408, 8175, 4, 0, 0, 0, 8159, 2, 15, 0, 0, 7694, 0, 8192, 0, 8191, 1, 3306, 1] payload_rows_per_dst=[16384, 16384, 16366, 10816, 16318, 16376, 15388, 16382] dup_rows_per_dst=[4, 2, 20, 8, 0, 8, 0, 6616] combine_rows_per_src=[16384, 16384, 16366, 10816, 16318, 16376, 15388, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=934428672 dispatch_meta_bytes=461208 combine_payload_bytes=934428672 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934889880 ingress_bytes=1286750208 link_bytes=1286750208 token_bytes=1869318552 weight_movement_bytes=352321536 total_nvlink_bytes=2221640088 weight_to_token_ratio=0.188476 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.205264049e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=934264832 dispatch_meta_bytes=458328 combine_payload_bytes=934264832 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=934723160 ingress_bytes=1815068672 link_bytes=1815068672 token_bytes=1868987992 weight_movement_bytes=880803840 total_nvlink_bytes=2749791832 weight_to_token_ratio=0.471273 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.016362692e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=925794304 dispatch_meta_bytes=470896 combine_payload_bytes=925794304 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=926265200 ingress_bytes=1982758912 link_bytes=1982758912 token_bytes=1852059504 weight_movement_bytes=1056964608 total_nvlink_bytes=2909024112 weight_to_token_ratio=0.570697 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=28672 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.110262226e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116764 payload_rows_remote=100380 dup_rows=14308 dup_rows_remote=460 dedup_ratio=0.109161 router_tokens_per_expert=[0, 0, 8186, 8192, 1, 0, 0, 8192, 0, 7223, 2, 0, 0, 0, 8190, 0, 0, 1, 12, 0, 8192, 0, 2237, 4, 8, 5, 0, 0, 0, 8192, 6897, 2] payload_rows_per_dst=[16322, 16384, 14450, 16380, 14698, 5762, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 24, 408, 26, 13848] combine_rows_per_src=[16322, 16384, 14450, 16380, 14698, 5762, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=885522432 dispatch_meta_bytes=459016 combine_payload_bytes=885522432 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=885981448 ingress_bytes=1237843968 link_bytes=1237843968 token_bytes=1771503880 weight_movement_bytes=352321536 total_nvlink_bytes=2123825416 weight_to_token_ratio=0.198883 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.931409519e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127764 payload_rows_remote=111380 dup_rows=3308 dup_rows_remote=3270 dedup_ratio=0.025238 router_tokens_per_expert=[8183, 3, 3, 0, 1, 0, 18, 8188, 303, 1, 0, 362, 8183, 0, 0, 7795, 0, 8191, 8191, 0, 7903, 0, 0, 0, 1, 18, 1, 0, 0, 8190, 0, 1] payload_rows_per_dst=[16374, 16384, 14292, 15760, 16382, 15806, 16384, 16382] dup_rows_per_dst=[4, 30, 908, 2328, 0, 0, 38, 0] combine_rows_per_src=[16374, 16384, 14292, 15760, 16382, 15806, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119164 payload_rows_remote=102780 dup_rows=11908 dup_rows_remote=10774 dedup_ratio=0.090851 router_tokens_per_expert=[0, 2, 8191, 8185, 3, 0, 0, 8028, 0, 5123, 0, 2, 3, 6, 8186, 0, 5, 0, 8, 3, 8188, 15, 6633, 0, 50, 29, 0, 0, 2, 8189, 4685, 0] payload_rows_per_dst=[16384, 16350, 16380, 16384, 16384, 13296, 15678, 8308] dup_rows_per_dst=[1134, 80, 10248, 336, 24, 0, 86, 0] combine_rows_per_src=[16384, 16350, 16380, 16384, 16384, 13296, 15678, 8308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 2, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121810 payload_rows_remote=110926 dup_rows=9262 dup_rows_remote=9236 dedup_ratio=0.070663 router_tokens_per_expert=[0, 0, 8090, 8192, 0, 0, 0, 8163, 3, 5124, 10, 0, 0, 14, 8188, 0, 0, 62, 186, 0, 8172, 0, 4624, 60, 1, 10, 0, 0, 0, 8192, 6445, 0] payload_rows_per_dst=[16180, 16326, 10884, 16378, 16384, 16384, 16384, 12890] dup_rows_per_dst=[0, 0, 26, 26, 456, 8732, 22, 0] combine_rows_per_src=[16180, 16326, 10884, 16378, 16384, 16384, 16384, 12890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126464 payload_rows_remote=113908 dup_rows=4608 dup_rows_remote=4578 dedup_ratio=0.035156 router_tokens_per_expert=[6, 25, 8123, 8152, 47, 1, 0, 8150, 1, 7958, 2, 25, 12, 0, 8091, 0, 7, 33, 343, 12, 8168, 9, 6269, 15, 24, 6, 4, 0, 20, 8182, 1851, 0] payload_rows_per_dst=[16286, 16372, 15964, 16206, 16382, 12556, 16314, 16384] dup_rows_per_dst=[22, 24, 8, 0, 744, 30, 58, 3722] combine_rows_per_src=[16286, 16372, 15964, 16206, 16382, 12556, 16314, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117646 payload_rows_remote=101290 dup_rows=13426 dup_rows_remote=13416 dedup_ratio=0.102432 router_tokens_per_expert=[0, 0, 7969, 8192, 0, 0, 24, 8159, 205, 6780, 1, 0, 0, 0, 8192, 0, 0, 14, 0, 0, 8145, 0, 7084, 80, 1, 184, 0, 0, 0, 8191, 2315, 0] payload_rows_per_dst=[15938, 16356, 15608, 16384, 16294, 16052, 16384, 4630] dup_rows_per_dst=[0, 10, 9730, 0, 24, 3292, 370, 0] combine_rows_per_src=[15938, 16356, 15608, 16384, 16294, 16052, 16384, 4630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121038 payload_rows_remote=104826 dup_rows=10034 dup_rows_remote=10034 dedup_ratio=0.076553 router_tokens_per_expert=[0, 0, 8139, 8190, 2, 0, 1, 8189, 4, 5921, 1, 0, 0, 0, 8106, 0, 0, 0, 121, 0, 8191, 1, 4829, 10, 0, 56, 2, 2, 0, 8190, 5581, 0] payload_rows_per_dst=[16278, 16384, 11852, 16212, 16382, 16384, 16384, 11162] dup_rows_per_dst=[0, 0, 0, 0, 242, 9676, 116, 0] combine_rows_per_src=[16278, 16384, 11852, 16212, 16382, 16384, 16384, 11162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=822312960 dispatch_meta_bytes=403360 combine_payload_bytes=822312960 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=822716320 ingress_bytes=2583920640 link_bytes=2583920640 token_bytes=1645029280 weight_movement_bytes=1761607680 total_nvlink_bytes=3406636960 weight_to_token_ratio=1.070867 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.446887700e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=912424960 dispatch_meta_bytes=458600 combine_payload_bytes=912424960 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=912883560 ingress_bytes=1264746496 link_bytes=1264746496 token_bytes=1825308520 weight_movement_bytes=352321536 total_nvlink_bytes=2177630056 weight_to_token_ratio=0.193020 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.082052446e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=841973760 dispatch_meta_bytes=454216 combine_payload_bytes=841973760 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=842427976 ingress_bytes=1546616832 link_bytes=1546616832 token_bytes=1684401736 weight_movement_bytes=704643072 total_nvlink_bytes=2389044808 weight_to_token_ratio=0.418334 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.660408670e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=908705792 dispatch_meta_bytes=480648 combine_payload_bytes=908705792 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=909186440 ingress_bytes=1261027328 link_bytes=1261027328 token_bytes=1817892232 weight_movement_bytes=352321536 total_nvlink_bytes=2170213768 weight_to_token_ratio=0.193808 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=27136 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.061226658e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=933134336 dispatch_meta_bytes=473944 combine_payload_bytes=933134336 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=933608280 ingress_bytes=1637777408 link_bytes=1637777408 token_bytes=1866742616 weight_movement_bytes=704643072 total_nvlink_bytes=2571385688 weight_to_token_ratio=0.377472 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=22912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.170869843e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=829767680 dispatch_meta_bytes=458824 combine_payload_bytes=829767680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=830226504 ingress_bytes=1182089216 link_bytes=1182089216 token_bytes=1659994184 weight_movement_bytes=352321536 total_nvlink_bytes=2012315720 weight_to_token_ratio=0.212243 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.619206181e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=858734592 dispatch_meta_bytes=459440 combine_payload_bytes=858734592 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=859194032 ingress_bytes=1211056128 link_bytes=1211056128 token_bytes=1717928624 weight_movement_bytes=352321536 total_nvlink_bytes=2070250160 weight_to_token_ratio=0.205085 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.781408797e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126388 payload_rows_remote=110008 dup_rows=4684 dup_rows_remote=4682 dedup_ratio=0.035736 router_tokens_per_expert=[8191, 0, 0, 0, 0, 0, 53, 8192, 12, 0, 0, 2267, 8192, 0, 0, 7183, 1, 8190, 8189, 0, 6864, 0, 0, 0, 0, 10, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 14366, 16380, 13728, 16380, 16384] dup_rows_per_dst=[0, 106, 4558, 0, 2, 0, 18, 0] combine_rows_per_src=[16382, 16384, 16384, 14366, 16380, 13728, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127852 payload_rows_remote=111470 dup_rows=3220 dup_rows_remote=2782 dedup_ratio=0.024567 router_tokens_per_expert=[8192, 4, 0, 106, 0, 1, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 1, 8079, 8178, 8191, 0, 0, 0, 0, 15, 8192, 0, 1, 0, 8192, 0, 8191, 0] payload_rows_per_dst=[13906, 16326, 16384, 16334, 16134, 16034, 16352, 16382] dup_rows_per_dst=[220, 0, 0, 2, 2528, 30, 2, 438] combine_rows_per_src=[13906, 16326, 16384, 16334, 16134, 16034, 16352, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121116 payload_rows_remote=104732 dup_rows=9956 dup_rows_remote=9890 dedup_ratio=0.075958 router_tokens_per_expert=[0, 1, 8138, 8190, 5, 0, 3, 8191, 0, 6036, 0, 0, 0, 1, 8192, 0, 0, 24, 100, 1, 8188, 2, 5424, 6, 19, 15, 1, 0, 0, 8191, 4808, 0] payload_rows_per_dst=[16278, 16384, 12072, 16384, 16380, 10852, 16384, 16382] dup_rows_per_dst=[0, 14, 0, 2, 246, 12, 66, 9616] combine_rows_per_src=[16278, 16384, 12072, 16384, 16380, 10852, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=901185536 dispatch_meta_bytes=458760 combine_payload_bytes=901185536 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=901644296 ingress_bytes=1958150144 link_bytes=1958150144 token_bytes=1802829832 weight_movement_bytes=1056964608 total_nvlink_bytes=2859794440 weight_to_token_ratio=0.586281 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.096482343e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129832 payload_rows_remote=113454 dup_rows=1240 dup_rows_remote=530 dedup_ratio=0.009460 router_tokens_per_expert=[8188, 0, 5, 103, 3, 7, 3, 3, 2, 1, 0, 8175, 0, 0, 0, 0, 8088, 8135, 8189, 1, 2, 2, 3, 38, 8187, 0, 17, 4, 8188, 3, 8184, 5] payload_rows_per_dst=[16378, 16380, 16284, 16378, 15814, 16262, 16382, 15954] dup_rows_per_dst=[710, 428, 0, 52, 0, 10, 34, 6] combine_rows_per_src=[16378, 16380, 16284, 16378, 15814, 16262, 16382, 15954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114654 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000259 router_tokens_per_expert=[8192, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 1, 6, 92, 0, 8189, 8188, 8092, 0, 0, 0, 0, 10, 8186, 0, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16370, 16376, 16384, 16372, 16384] dup_rows_per_dst=[4, 4, 0, 12, 0, 14, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16370, 16376, 16384, 16372, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=913162240 dispatch_meta_bytes=457008 combine_payload_bytes=913162240 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=913619248 ingress_bytes=1793966080 link_bytes=1793966080 token_bytes=1826781488 weight_movement_bytes=880803840 total_nvlink_bytes=2707585328 weight_to_token_ratio=0.482162 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.004546121e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129644 payload_rows_remote=113502 dup_rows=1428 dup_rows_remote=1428 dedup_ratio=0.010895 router_tokens_per_expert=[8191, 74, 1, 479, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 114, 0, 8012, 7690, 8092, 0, 0, 1, 0, 111, 8179, 0, 6, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16384, 16142, 16384, 16342, 15262, 16384, 16362, 16384] dup_rows_per_dst=[1106, 0, 0, 90, 0, 224, 8, 0] combine_rows_per_src=[16384, 16142, 16384, 16342, 15262, 16384, 16362, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130920 payload_rows_remote=114544 dup_rows=152 dup_rows_remote=126 dedup_ratio=0.001160 router_tokens_per_expert=[8190, 6, 0, 22, 0, 0, 2, 2, 0, 0, 0, 8186, 2, 0, 31, 0, 8134, 8185, 8168, 2, 1, 2, 0, 32, 8191, 0, 0, 0, 8190, 0, 8190, 0] payload_rows_per_dst=[16384, 16336, 16372, 16376, 16306, 16384, 16382, 16380] dup_rows_per_dst=[52, 4, 0, 26, 0, 70, 0, 0] combine_rows_per_src=[16384, 16336, 16372, 16376, 16306, 16384, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130346 payload_rows_remote=114134 dup_rows=726 dup_rows_remote=568 dedup_ratio=0.005539 router_tokens_per_expert=[8170, 0, 1, 205, 9, 19, 5, 0, 22, 52, 0, 8173, 0, 7, 6, 0, 8012, 7970, 8144, 3, 28, 10, 4, 172, 8140, 11, 0, 14, 8171, 16, 8162, 10] payload_rows_per_dst=[16356, 16300, 16384, 16308, 16110, 16212, 16320, 16356] dup_rows_per_dst=[396, 26, 110, 6, 0, 158, 10, 20] combine_rows_per_src=[16356, 16300, 16384, 16308, 16110, 16212, 16320, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=857964544 dispatch_meta_bytes=458488 combine_payload_bytes=857964544 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=858423032 ingress_bytes=1210286080 link_bytes=1210286080 token_bytes=1716387576 weight_movement_bytes=352321536 total_nvlink_bytes=2068709112 weight_to_token_ratio=0.205269 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.777096850e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=929415168 dispatch_meta_bytes=455936 combine_payload_bytes=929415168 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=929871104 ingress_bytes=1634058240 link_bytes=1634058240 token_bytes=1859286272 weight_movement_bytes=704643072 total_nvlink_bytes=2563929344 weight_to_token_ratio=0.378986 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.150044055e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=458752 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939704320 ingress_bytes=1291567104 link_bytes=1291567104 token_bytes=1878949888 weight_movement_bytes=352321536 total_nvlink_bytes=2231271424 weight_to_token_ratio=0.187510 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232236656e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=929808384 dispatch_meta_bytes=459720 combine_payload_bytes=929808384 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=930268104 ingress_bytes=1458290688 link_bytes=1458290688 token_bytes=1860076488 weight_movement_bytes=528482304 total_nvlink_bytes=2388558792 weight_to_token_ratio=0.284119 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.165819132e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938344448 dispatch_meta_bytes=458680 combine_payload_bytes=938344448 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=938803128 ingress_bytes=1466826752 link_bytes=1466826752 token_bytes=1877147576 weight_movement_bytes=528482304 total_nvlink_bytes=2405629880 weight_to_token_ratio=0.281535 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.213617528e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=934985728 dispatch_meta_bytes=458808 combine_payload_bytes=934985728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935444536 ingress_bytes=1287307264 link_bytes=1287307264 token_bytes=1870430264 weight_movement_bytes=352321536 total_nvlink_bytes=2222751800 weight_to_token_ratio=0.188364 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.208383330e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118512 payload_rows_remote=102128 dup_rows=12560 dup_rows_remote=12536 dedup_ratio=0.095825 router_tokens_per_expert=[0, 0, 8125, 8192, 0, 0, 1, 8192, 0, 6925, 0, 0, 0, 0, 8190, 0, 0, 0, 12, 0, 8192, 0, 4966, 11, 1, 15, 0, 0, 0, 8192, 4522, 0] payload_rows_per_dst=[16250, 16384, 13850, 16380, 16384, 11384, 16384, 11496] dup_rows_per_dst=[0, 2, 0, 0, 24, 6252, 32, 6250] combine_rows_per_src=[16250, 16384, 13850, 16380, 16384, 11384, 16384, 11496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117068 payload_rows_remote=100890 dup_rows=14004 dup_rows_remote=360 dedup_ratio=0.106842 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8191, 17, 267, 0, 8189, 2, 0, 8192, 2, 0, 0, 0, 8192, 0, 0, 1, 0, 8190, 0, 0, 8192, 0, 0, 7909, 0] payload_rows_per_dst=[16384, 16206, 2912, 16384, 16380, 16244, 16380, 16178] dup_rows_per_dst=[0, 0, 204, 10, 0, 2, 144, 13644] combine_rows_per_src=[16384, 16206, 2912, 16384, 16380, 16244, 16380, 16178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130890 payload_rows_remote=114506 dup_rows=182 dup_rows_remote=156 dedup_ratio=0.001389 router_tokens_per_expert=[8191, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 8192, 1, 0, 3, 4, 8105, 8189, 8185, 0, 0, 0, 0, 75, 8192, 0, 13, 0, 8191, 1, 8191, 0] payload_rows_per_dst=[16382, 16384, 16384, 16378, 16212, 16382, 16384, 16384] dup_rows_per_dst=[2, 0, 2, 8, 0, 144, 26, 0] combine_rows_per_src=[16382, 16384, 16384, 16378, 16212, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=836632576 dispatch_meta_bytes=458656 combine_payload_bytes=836632576 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=837091232 ingress_bytes=1188954112 link_bytes=1188954112 token_bytes=1673723808 weight_movement_bytes=352321536 total_nvlink_bytes=2026045344 weight_to_token_ratio=0.210502 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.657646733e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130048 payload_rows_remote=113668 dup_rows=1024 dup_rows_remote=968 dedup_ratio=0.007812 router_tokens_per_expert=[8187, 3, 0, 1, 0, 0, 2, 8188, 19, 393, 0, 8188, 0, 0, 8185, 0, 3, 0, 0, 8180, 5, 14, 4, 0, 8189, 2, 6, 8187, 0, 4, 7774, 2] payload_rows_per_dst=[16380, 16380, 16376, 16370, 16346, 16384, 16256, 15556] dup_rows_per_dst=[56, 24, 832, 60, 4, 40, 4, 4] combine_rows_per_src=[16380, 16380, 16376, 16370, 16346, 16384, 16256, 15556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124134 payload_rows_remote=107752 dup_rows=6938 dup_rows_remote=42 dedup_ratio=0.052933 router_tokens_per_expert=[8188, 0, 0, 0, 6, 0, 0, 8192, 45, 3418, 0, 8176, 8, 0, 8192, 0, 0, 4, 0, 8192, 0, 0, 0, 3, 8187, 0, 0, 8192, 0, 0, 4733, 0] payload_rows_per_dst=[16376, 16384, 16382, 16384, 16384, 16382, 16376, 9466] dup_rows_per_dst=[0, 12, 6896, 16, 8, 6, 0, 0] combine_rows_per_src=[16376, 16384, 16382, 16384, 16384, 16382, 16376, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=826490880 dispatch_meta_bytes=405000 combine_payload_bytes=826490880 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=826895880 ingress_bytes=1178812416 link_bytes=1178812416 token_bytes=1653386760 weight_movement_bytes=352321536 total_nvlink_bytes=2005708296 weight_to_token_ratio=0.213091 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.600857469e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130668 payload_rows_remote=114284 dup_rows=404 dup_rows_remote=262 dedup_ratio=0.003082 router_tokens_per_expert=[8192, 0, 0, 0, 71, 0, 0, 8192, 1, 129, 0, 8168, 0, 0, 8180, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8026, 0] payload_rows_per_dst=[16384, 16384, 16336, 16360, 16384, 16384, 16384, 16052] dup_rows_per_dst=[0, 142, 260, 0, 2, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16336, 16360, 16384, 16384, 16384, 16052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129652 payload_rows_remote=113272 dup_rows=1420 dup_rows_remote=1418 dedup_ratio=0.010834 router_tokens_per_expert=[8190, 0, 0, 0, 2, 0, 0, 8190, 0, 701, 0, 8189, 0, 2, 8189, 0, 0, 5, 2, 8192, 0, 0, 0, 0, 8191, 2, 0, 8190, 1, 0, 7490, 0] payload_rows_per_dst=[16380, 16380, 16382, 16380, 16384, 16382, 16384, 14980] dup_rows_per_dst=[0, 4, 1398, 2, 14, 0, 0, 2] combine_rows_per_src=[16380, 16380, 16382, 16380, 16384, 16382, 16384, 14980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938033152 dispatch_meta_bytes=458648 combine_payload_bytes=938033152 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=938491800 ingress_bytes=1114193920 link_bytes=1114193920 token_bytes=1876524952 weight_movement_bytes=176160768 total_nvlink_bytes=2052685720 weight_to_token_ratio=0.093876 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239020865e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128842 payload_rows_remote=112458 dup_rows=2230 dup_rows_remote=2134 dedup_ratio=0.017014 router_tokens_per_expert=[8161, 0, 13, 0, 13, 0, 9, 8170, 1, 1105, 10, 8085, 3, 0, 8167, 0, 9, 0, 0, 8167, 19, 25, 15, 0, 8181, 3, 20, 8165, 0, 20, 7150, 25] payload_rows_per_dst=[16348, 16358, 16360, 16340, 16352, 16384, 16370, 14330] dup_rows_per_dst=[0, 26, 2042, 0, 0, 96, 6, 60] combine_rows_per_src=[16348, 16358, 16360, 16340, 16352, 16384, 16370, 14330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=931168256 dispatch_meta_bytes=458544 combine_payload_bytes=931168256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=931626800 ingress_bytes=1283489792 link_bytes=1283489792 token_bytes=1862795056 weight_movement_bytes=352321536 total_nvlink_bytes=2215116592 weight_to_token_ratio=0.189136 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.187007080e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=882704384 dispatch_meta_bytes=431176 combine_payload_bytes=882704384 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=883135560 ingress_bytes=1587347456 link_bytes=1587347456 token_bytes=1765839944 weight_movement_bytes=704643072 total_nvlink_bytes=2470483016 weight_to_token_ratio=0.399041 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=29440 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.888483162e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=936214528 dispatch_meta_bytes=458184 combine_payload_bytes=936214528 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=936672712 ingress_bytes=1464696832 link_bytes=1464696832 token_bytes=1872887240 weight_movement_bytes=528482304 total_nvlink_bytes=2401369544 weight_to_token_ratio=0.282175 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.201690865e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=927924224 dispatch_meta_bytes=458760 combine_payload_bytes=927924224 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928382984 ingress_bytes=1280245760 link_bytes=1280245760 token_bytes=1856307208 weight_movement_bytes=352321536 total_nvlink_bytes=2208628744 weight_to_token_ratio=0.189797 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.168841855e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=921255936 dispatch_meta_bytes=458368 combine_payload_bytes=921255936 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=921714304 ingress_bytes=1273577472 link_bytes=1273577472 token_bytes=1842970240 weight_movement_bytes=352321536 total_nvlink_bytes=2195291776 weight_to_token_ratio=0.191170 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.131502226e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127788 payload_rows_remote=111408 dup_rows=3284 dup_rows_remote=2750 dedup_ratio=0.025055 router_tokens_per_expert=[8192, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 8192, 0, 3, 2, 8192, 8192, 7543, 0, 0, 610, 0, 0, 8192, 0, 8190] payload_rows_per_dst=[16356, 16076, 16304, 16358, 14840, 15090, 16384, 16380] dup_rows_per_dst=[0, 66, 0, 0, 12, 1452, 1220, 534] combine_rows_per_src=[16356, 16076, 16304, 16358, 14840, 15090, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130984 payload_rows_remote=114654 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.000671 router_tokens_per_expert=[8192, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 8167, 8173, 8174, 0, 0, 0, 0, 2, 8191, 0, 28, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16384, 16358, 16384, 16382, 16330, 16378, 16384, 16384] dup_rows_per_dst=[24, 0, 0, 8, 0, 2, 54, 0] combine_rows_per_src=[16384, 16358, 16384, 16382, 16330, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130422 payload_rows_remote=114038 dup_rows=650 dup_rows_remote=650 dedup_ratio=0.004959 router_tokens_per_expert=[8190, 0, 0, 0, 3, 0, 0, 8191, 10, 295, 0, 8187, 0, 1, 8191, 0, 0, 19, 1, 8191, 1, 0, 0, 0, 8191, 0, 1, 8191, 0, 1, 7872, 0] payload_rows_per_dst=[16380, 16384, 16380, 16382, 16384, 16384, 16384, 15744] dup_rows_per_dst=[0, 4, 604, 2, 38, 0, 0, 2] combine_rows_per_src=[16380, 16384, 16380, 16382, 16384, 16384, 16384, 15744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 3, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123468 payload_rows_remote=107090 dup_rows=7604 dup_rows_remote=7580 dedup_ratio=0.058014 router_tokens_per_expert=[8180, 3, 8, 0, 5, 0, 0, 3, 0, 0, 5, 0, 8189, 2, 11, 0, 34, 8184, 4, 0, 2, 8188, 8186, 7918, 2, 0, 233, 5, 0, 8188, 0, 8186] payload_rows_per_dst=[16378, 16382, 16382, 16376, 16376, 16368, 9120, 16086] dup_rows_per_dst=[24, 10, 0, 8, 68, 7480, 14, 0] combine_rows_per_src=[16378, 16382, 16382, 16376, 16376, 16368, 9120, 16086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=912654336 dispatch_meta_bytes=456632 combine_payload_bytes=912654336 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=913110968 ingress_bytes=1793458176 link_bytes=1793458176 token_bytes=1825765304 weight_movement_bytes=880803840 total_nvlink_bytes=2706569144 weight_to_token_ratio=0.482430 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.004261716e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=458968 combine_payload_bytes=939245568 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=939704536 ingress_bytes=2348531712 link_bytes=2348531712 token_bytes=1878950104 weight_movement_bytes=1409286144 total_nvlink_bytes=3288236248 weight_to_token_ratio=0.750039 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315079726e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127804 payload_rows_remote=111420 dup_rows=3268 dup_rows_remote=3260 dedup_ratio=0.024933 router_tokens_per_expert=[8192, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 9, 157, 8192, 0, 2, 4, 8192, 8192, 6554, 0, 0, 1462, 0, 0, 8191, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 13116, 16384, 16384] dup_rows_per_dst=[0, 8, 0, 18, 318, 0, 2924, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 13116, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129222 payload_rows_remote=112838 dup_rows=1850 dup_rows_remote=1828 dedup_ratio=0.014114 router_tokens_per_expert=[8188, 0, 0, 0, 45, 0, 0, 0, 0, 11, 0, 0, 8187, 0, 0, 14, 450, 8192, 0, 34, 0, 8189, 8192, 7325, 0, 0, 372, 0, 0, 8192, 3, 8142] payload_rows_per_dst=[16376, 16384, 16384, 16378, 16384, 14650, 16378, 16288] dup_rows_per_dst=[0, 90, 22, 24, 968, 0, 744, 2] combine_rows_per_src=[16376, 16384, 16384, 16378, 16384, 14650, 16378, 16288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=934199296 dispatch_meta_bytes=458752 combine_payload_bytes=934199296 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=934658048 ingress_bytes=2167324672 link_bytes=2167324672 token_bytes=1868857344 weight_movement_bytes=1233125376 total_nvlink_bytes=3101982720 weight_to_token_ratio=0.659829 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.213611348e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130398 payload_rows_remote=114014 dup_rows=674 dup_rows_remote=674 dedup_ratio=0.005142 router_tokens_per_expert=[8187, 0, 0, 0, 5, 2, 0, 0, 2, 0, 1, 0, 8192, 0, 0, 0, 89, 8190, 0, 1, 9, 8185, 8190, 7863, 0, 0, 244, 0, 1, 8189, 0, 8186] payload_rows_per_dst=[16374, 16384, 16384, 16384, 16382, 15740, 16378, 16372] dup_rows_per_dst=[0, 8, 2, 0, 178, 4, 480, 2] combine_rows_per_src=[16374, 16384, 16384, 16384, 16382, 15740, 16378, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130468 payload_rows_remote=114490 dup_rows=604 dup_rows_remote=604 dedup_ratio=0.004608 router_tokens_per_expert=[8172, 5, 25, 4, 64, 0, 0, 24, 0, 3, 24, 22, 8168, 3, 4, 2, 2, 8156, 18, 3, 8, 8158, 8166, 7981, 16, 0, 190, 20, 0, 8165, 0, 8133] payload_rows_per_dst=[16384, 16384, 16384, 16346, 16348, 15978, 16378, 16266] dup_rows_per_dst=[28, 122, 46, 8, 10, 0, 390, 0] combine_rows_per_src=[16384, 16384, 16384, 16346, 16348, 15978, 16378, 16266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=877281280 dispatch_meta_bytes=458680 combine_payload_bytes=877281280 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=877739960 ingress_bytes=1405763584 link_bytes=1405763584 token_bytes=1755021240 weight_movement_bytes=528482304 total_nvlink_bytes=2283503544 weight_to_token_ratio=0.301126 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.871689276e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=912752640 dispatch_meta_bytes=458720 combine_payload_bytes=912752640 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=913211360 ingress_bytes=1265074176 link_bytes=1265074176 token_bytes=1825964000 weight_movement_bytes=352321536 total_nvlink_bytes=2178285536 weight_to_token_ratio=0.192951 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.083887318e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=924368896 dispatch_meta_bytes=458664 combine_payload_bytes=924368896 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=924827560 ingress_bytes=1276690432 link_bytes=1276690432 token_bytes=1849196456 weight_movement_bytes=352321536 total_nvlink_bytes=2201517992 weight_to_token_ratio=0.190527 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.148933502e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=934002688 dispatch_meta_bytes=458752 combine_payload_bytes=934002688 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=934461440 ingress_bytes=1462484992 link_bytes=1462484992 token_bytes=1868464128 weight_movement_bytes=528482304 total_nvlink_bytes=2396946432 weight_to_token_ratio=0.282843 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.189305484e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937902080 dispatch_meta_bytes=460376 combine_payload_bytes=937902080 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=938362456 ingress_bytes=1994866688 link_bytes=1994866688 token_bytes=1876264536 weight_movement_bytes=1056964608 total_nvlink_bytes=2933229144 weight_to_token_ratio=0.563335 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117042075e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126018 payload_rows_remote=109634 dup_rows=5054 dup_rows_remote=5014 dedup_ratio=0.038559 router_tokens_per_expert=[0, 0, 159, 8192, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 8033, 0, 8190, 0, 0, 0, 8192, 8191, 0, 0, 8192, 2, 0, 8192, 0, 0, 0] payload_rows_per_dst=[11786, 16336, 16384, 16066, 16296, 16382, 16384, 16384] dup_rows_per_dst=[318, 0, 62, 4028, 14, 544, 48, 40] combine_rows_per_src=[11786, 16336, 16384, 16066, 16296, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129738 payload_rows_remote=113354 dup_rows=1334 dup_rows_remote=1326 dedup_ratio=0.010178 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 58, 605, 0, 8191, 1, 0, 8190, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 7527, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 15054] dup_rows_per_dst=[0, 0, 1324, 2, 8, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 15054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130234 payload_rows_remote=113854 dup_rows=838 dup_rows_remote=36 dedup_ratio=0.006393 router_tokens_per_expert=[8191, 0, 1, 0, 1, 0, 2, 0, 1, 4, 1, 0, 8189, 0, 0, 5, 10, 8187, 1, 2, 0, 8190, 8191, 7778, 0, 0, 401, 0, 0, 8191, 0, 8190] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 15556, 16380, 16380] dup_rows_per_dst=[0, 4, 10, 6, 16, 0, 802, 0] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 15556, 16380, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=898121728 dispatch_meta_bytes=458592 combine_payload_bytes=898121728 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=898580320 ingress_bytes=1074282496 link_bytes=1074282496 token_bytes=1796702048 weight_movement_bytes=176160768 total_nvlink_bytes=1972862816 weight_to_token_ratio=0.098047 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.015533550e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130764 payload_rows_remote=114392 dup_rows=308 dup_rows_remote=250 dedup_ratio=0.002350 router_tokens_per_expert=[0, 0, 10, 8186, 0, 4, 5, 6, 0, 2, 0, 8186, 92, 5, 0, 8135, 0, 8186, 0, 10, 0, 8185, 8135, 2, 0, 8187, 0, 3, 8187, 0, 6, 4] payload_rows_per_dst=[16372, 16382, 16376, 16380, 16356, 16150, 16364, 16384] dup_rows_per_dst=[58, 142, 6, 84, 8, 0, 0, 10] combine_rows_per_src=[16372, 16382, 16376, 16380, 16356, 16150, 16364, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128204 payload_rows_remote=111854 dup_rows=2868 dup_rows_remote=2868 dedup_ratio=0.021881 router_tokens_per_expert=[0, 0, 1417, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6882, 4, 8192, 0, 0, 0, 8175, 8085, 0, 13, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16350, 16384, 13764, 16384, 16170, 16384, 16384] dup_rows_per_dst=[2834, 0, 0, 0, 8, 0, 26, 0] combine_rows_per_src=[16384, 16350, 16384, 13764, 16384, 16170, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=928595968 dispatch_meta_bytes=458720 combine_payload_bytes=928595968 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=929054688 ingress_bytes=1633239040 link_bytes=1633239040 token_bytes=1857650656 weight_movement_bytes=704643072 total_nvlink_bytes=2562293728 weight_to_token_ratio=0.379319 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.145456876e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130666 payload_rows_remote=114290 dup_rows=406 dup_rows_remote=406 dedup_ratio=0.003098 router_tokens_per_expert=[1, 2, 197, 8192, 0, 0, 0, 0, 0, 1, 0, 8187, 0, 0, 0, 8040, 2, 8192, 0, 0, 0, 8148, 8189, 0, 0, 8192, 0, 0, 8192, 1, 0, 0] payload_rows_per_dst=[16384, 16296, 16376, 16080, 16384, 16378, 16384, 16384] dup_rows_per_dst=[400, 0, 0, 0, 4, 0, 0, 2] combine_rows_per_src=[16384, 16296, 16376, 16080, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130506 payload_rows_remote=114650 dup_rows=566 dup_rows_remote=566 dedup_ratio=0.004318 router_tokens_per_expert=[0, 0, 282, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 7926, 0, 8190, 0, 0, 0, 8192, 8189, 0, 0, 8176, 3, 1, 8191, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 15856, 16380, 16378, 16358, 16382] dup_rows_per_dst=[564, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 15856, 16380, 16378, 16358, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=932691968 dispatch_meta_bytes=455560 combine_payload_bytes=932691968 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933147528 ingress_bytes=1285013504 link_bytes=1285013504 token_bytes=1865839496 weight_movement_bytes=352321536 total_nvlink_bytes=2218161032 weight_to_token_ratio=0.188827 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.195539231e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130560 payload_rows_remote=114238 dup_rows=512 dup_rows_remote=512 dedup_ratio=0.003906 router_tokens_per_expert=[0, 0, 99, 8168, 39, 25, 20, 18, 0, 7, 1, 8159, 77, 22, 0, 8057, 20, 8172, 2, 34, 33, 8111, 8114, 14, 0, 8154, 0, 22, 8164, 0, 1, 3] payload_rows_per_dst=[16336, 16306, 16332, 16192, 16384, 16322, 16352, 16336] dup_rows_per_dst=[198, 120, 2, 120, 72, 0, 0, 0] combine_rows_per_src=[16336, 16306, 16332, 16192, 16384, 16322, 16352, 16336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 6, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937099264 dispatch_meta_bytes=458568 combine_payload_bytes=937099264 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=937557832 ingress_bytes=2170224640 link_bytes=2170224640 token_bytes=1874657096 weight_movement_bytes=1233125376 total_nvlink_bytes=3107782472 weight_to_token_ratio=0.657787 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.215235209e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 3, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=916307968 dispatch_meta_bytes=458888 combine_payload_bytes=916307968 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=916766856 ingress_bytes=1444790272 link_bytes=1444790272 token_bytes=1833074824 weight_movement_bytes=528482304 total_nvlink_bytes=2361557128 weight_to_token_ratio=0.288304 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.090222438e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936263680 dispatch_meta_bytes=458784 combine_payload_bytes=936263680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936722464 ingress_bytes=1288585216 link_bytes=1288585216 token_bytes=1872986144 weight_movement_bytes=352321536 total_nvlink_bytes=2225307680 weight_to_token_ratio=0.188107 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.215539328e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=460864 combine_payload_bytes=939212800 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939673664 ingress_bytes=1291534336 link_bytes=1291534336 token_bytes=1878886464 weight_movement_bytes=352321536 total_nvlink_bytes=2231208000 weight_to_token_ratio=0.187516 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232053169e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=935837696 dispatch_meta_bytes=459000 combine_payload_bytes=935837696 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=936296696 ingress_bytes=1816641536 link_bytes=1816641536 token_bytes=1872134392 weight_movement_bytes=880803840 total_nvlink_bytes=2752938232 weight_to_token_ratio=0.470481 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.017243430e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128942 payload_rows_remote=112580 dup_rows=2130 dup_rows_remote=1466 dedup_ratio=0.016251 router_tokens_per_expert=[0, 8192, 25, 0, 0, 8180, 0, 8192, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 2, 8179, 0] payload_rows_per_dst=[14492, 16360, 16282, 16382, 16300, 16382, 16382, 16362] dup_rows_per_dst=[50, 1316, 2, 16, 0, 0, 82, 664] combine_rows_per_src=[14492, 16360, 16282, 16382, 16300, 16382, 16382, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130806 payload_rows_remote=114422 dup_rows=266 dup_rows_remote=264 dedup_ratio=0.002029 router_tokens_per_expert=[0, 2, 122, 8191, 0, 1, 1, 1, 0, 0, 0, 8191, 1, 1, 0, 8067, 6, 8191, 1, 1, 1, 8185, 8189, 0, 0, 8191, 1, 1, 8191, 0, 0, 0] payload_rows_per_dst=[16384, 16372, 16382, 16138, 16384, 16380, 16384, 16382] dup_rows_per_dst=[246, 4, 0, 0, 14, 0, 2, 0] combine_rows_per_src=[16384, 16372, 16382, 16138, 16384, 16380, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130320 payload_rows_remote=113936 dup_rows=752 dup_rows_remote=732 dedup_ratio=0.005737 router_tokens_per_expert=[8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 7, 1, 10, 8192, 0, 0, 7, 8192, 8192, 7812, 0, 0, 356, 0, 0, 8192, 0, 8189] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 15638, 16384, 16378] dup_rows_per_dst=[0, 2, 2, 16, 20, 0, 712, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 15638, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=922255360 dispatch_meta_bytes=456184 combine_payload_bytes=922255360 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=922711544 ingress_bytes=1626898432 link_bytes=1626898432 token_bytes=1844966904 weight_movement_bytes=704643072 total_nvlink_bytes=2549609976 weight_to_token_ratio=0.381927 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.109952118e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130824 payload_rows_remote=114442 dup_rows=248 dup_rows_remote=86 dedup_ratio=0.001892 router_tokens_per_expert=[2, 8187, 4, 5, 0, 8145, 41, 8186, 0, 8, 0, 5, 8184, 2, 3, 0, 20, 8167, 1, 0, 8, 0, 0, 4, 8183, 0, 4, 0, 8187, 0, 8188, 2] payload_rows_per_dst=[16382, 16374, 16380, 16374, 16228, 16332, 16374, 16380] dup_rows_per_dst=[162, 60, 16, 4, 0, 6, 0, 0] combine_rows_per_src=[16382, 16374, 16380, 16374, 16228, 16332, 16374, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129674 payload_rows_remote=114484 dup_rows=1398 dup_rows_remote=1398 dedup_ratio=0.010666 router_tokens_per_expert=[0, 8192, 654, 0, 0, 7595, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 45, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8090, 0] payload_rows_per_dst=[16384, 15190, 16384, 16384, 16384, 16384, 16384, 16180] dup_rows_per_dst=[1308, 0, 0, 0, 90, 0, 0, 0] combine_rows_per_src=[16384, 15190, 16384, 16384, 16384, 16384, 16384, 16180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937345024 dispatch_meta_bytes=458744 combine_payload_bytes=937345024 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=937803768 ingress_bytes=1465827328 link_bytes=1465827328 token_bytes=1875148792 weight_movement_bytes=528482304 total_nvlink_bytes=2403631096 weight_to_token_ratio=0.281835 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.208021170e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130908 payload_rows_remote=114524 dup_rows=164 dup_rows_remote=160 dedup_ratio=0.001251 router_tokens_per_expert=[0, 8180, 77, 0, 0, 8180, 12, 8192, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 8180, 0, 23, 0, 1, 0, 0, 8189, 0, 0, 1, 8192, 4, 8111, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16382, 16384, 16378, 16230] dup_rows_per_dst=[132, 0, 4, 0, 24, 2, 2, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16382, 16384, 16378, 16230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130870 payload_rows_remote=114488 dup_rows=202 dup_rows_remote=202 dedup_ratio=0.001541 router_tokens_per_expert=[0, 8192, 83, 0, 2, 8163, 1, 8190, 3, 0, 1, 0, 8191, 0, 0, 0, 0, 8190, 0, 15, 0, 0, 0, 1, 8192, 0, 0, 0, 8189, 0, 8121, 2] payload_rows_per_dst=[16384, 16328, 16384, 16382, 16380, 16382, 16384, 16246] dup_rows_per_dst=[166, 4, 2, 0, 30, 0, 0, 0] combine_rows_per_src=[16384, 16328, 16384, 16382, 16380, 16382, 16384, 16246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130826 payload_rows_remote=114442 dup_rows=246 dup_rows_remote=198 dedup_ratio=0.001877 router_tokens_per_expert=[13, 8156, 19, 21, 0, 7960, 218, 8166, 0, 43, 0, 22, 8154, 4, 24, 6, 28, 8164, 6, 0, 39, 0, 0, 11, 8142, 10, 6, 1, 8168, 0, 8155, 0] payload_rows_per_dst=[16356, 16348, 16384, 16356, 16372, 16384, 16316, 16310] dup_rows_per_dst=[62, 8, 82, 20, 24, 48, 2, 0] combine_rows_per_src=[16356, 16348, 16384, 16356, 16372, 16384, 16316, 16310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=933363712 dispatch_meta_bytes=458672 combine_payload_bytes=933363712 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=933822384 ingress_bytes=1814167552 link_bytes=1814167552 token_bytes=1867186096 weight_movement_bytes=880803840 total_nvlink_bytes=2747989936 weight_to_token_ratio=0.471728 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.015858102e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937508864 dispatch_meta_bytes=458112 combine_payload_bytes=937508864 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=937966976 ingress_bytes=2170634240 link_bytes=2170634240 token_bytes=1875475840 weight_movement_bytes=1233125376 total_nvlink_bytes=3108601216 weight_to_token_ratio=0.657500 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.215464568e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937852928 dispatch_meta_bytes=463528 combine_payload_bytes=937852928 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=938316456 ingress_bytes=1642496000 link_bytes=1642496000 token_bytes=1876169384 weight_movement_bytes=704643072 total_nvlink_bytes=2580812456 weight_to_token_ratio=0.375575 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.197291988e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938180608 dispatch_meta_bytes=458736 combine_payload_bytes=938180608 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938639344 ingress_bytes=1290502144 link_bytes=1290502144 token_bytes=1876819952 weight_movement_bytes=352321536 total_nvlink_bytes=2229141488 weight_to_token_ratio=0.187723 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226273324e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131002 payload_rows_remote=114618 dup_rows=70 dup_rows_remote=36 dedup_ratio=0.000534 router_tokens_per_expert=[0, 8192, 0, 0, 0, 1, 0, 0, 8185, 8192, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 8192, 0] payload_rows_per_dst=[16362, 16380, 16342, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 28, 6, 34] combine_rows_per_src=[16362, 16380, 16342, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937885696 dispatch_meta_bytes=458760 combine_payload_bytes=937885696 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938344456 ingress_bytes=1290207232 link_bytes=1290207232 token_bytes=1876230152 weight_movement_bytes=352321536 total_nvlink_bytes=2228551688 weight_to_token_ratio=0.187782 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224621940e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937508864 dispatch_meta_bytes=458560 combine_payload_bytes=937508864 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937967424 ingress_bytes=1289830400 link_bytes=1289830400 token_bytes=1875476288 weight_movement_bytes=352321536 total_nvlink_bytes=2227797824 weight_to_token_ratio=0.187857 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.222511838e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130984 payload_rows_remote=114602 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.000671 router_tokens_per_expert=[1, 8191, 19, 0, 0, 8159, 1, 8191, 0, 0, 0, 1, 8191, 1, 1, 0, 0, 8191, 1, 23, 1, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 8181, 1] payload_rows_per_dst=[16384, 16320, 16384, 16384, 16384, 16384, 16382, 16362] dup_rows_per_dst=[38, 0, 0, 2, 46, 0, 0, 2] combine_rows_per_src=[16384, 16320, 16384, 16384, 16384, 16384, 16382, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938950656 dispatch_meta_bytes=458616 combine_payload_bytes=938950656 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939409272 ingress_bytes=1643593728 link_bytes=1643593728 token_bytes=1878359928 weight_movement_bytes=704643072 total_nvlink_bytes=2583003000 weight_to_token_ratio=0.375137 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203438807e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130728 payload_rows_remote=114344 dup_rows=344 dup_rows_remote=326 dedup_ratio=0.002625 router_tokens_per_expert=[9, 8182, 0, 5, 5, 0, 1, 1, 8187, 8186, 8174, 5, 8185, 8186, 2, 2, 0, 7, 1, 0, 3, 1, 2, 0, 10, 9, 0, 8178, 3, 0, 8184, 8] payload_rows_per_dst=[16384, 16384, 16378, 16380, 16346, 16360, 16336, 16160] dup_rows_per_dst=[18, 18, 216, 82, 2, 0, 8, 0] combine_rows_per_src=[16384, 16384, 16378, 16380, 16346, 16360, 16336, 16160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114648 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 20, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8182, 0, 8192, 0, 0, 0, 8190, 8184, 0, 0, 8192, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16364, 16384, 16368, 16384, 16384] dup_rows_per_dst=[40, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16364, 16384, 16368, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131016 payload_rows_remote=114640 dup_rows=56 dup_rows_remote=54 dedup_ratio=0.000427 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 1, 0, 8169, 8188, 8191, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 27, 0, 8192, 0] payload_rows_per_dst=[16384, 16376, 16338, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 54] combine_rows_per_src=[16384, 16376, 16338, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938819584 dispatch_meta_bytes=458760 combine_payload_bytes=938819584 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939278344 ingress_bytes=1291141120 link_bytes=1291141120 token_bytes=1878097928 weight_movement_bytes=352321536 total_nvlink_bytes=2230419464 weight_to_token_ratio=0.187595 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229851323e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131022 payload_rows_remote=114688 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000381 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8167, 8192, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 25, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16334, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 50] combine_rows_per_src=[16384, 16384, 16334, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114674 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000107 router_tokens_per_expert=[0, 8192, 0, 0, 0, 1, 0, 0, 8189, 8192, 8189, 0, 8192, 8192, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 8191, 0] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16384, 16378, 16384, 16382] dup_rows_per_dst=[0, 2, 0, 0, 10, 0, 0, 2] combine_rows_per_src=[16384, 16384, 16378, 16384, 16384, 16378, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130912 payload_rows_remote=114562 dup_rows=160 dup_rows_remote=124 dedup_ratio=0.001221 router_tokens_per_expert=[34, 8147, 0, 15, 25, 0, 0, 0, 8165, 8160, 8139, 21, 8153, 8167, 0, 0, 0, 33, 5, 0, 0, 18, 36, 0, 64, 41, 0, 8107, 13, 17, 8149, 27] payload_rows_per_dst=[16382, 16370, 16372, 16306, 16384, 16350, 16380, 16368] dup_rows_per_dst=[10, 0, 0, 0, 26, 36, 44, 44] combine_rows_per_src=[16382, 16370, 16372, 16306, 16384, 16350, 16380, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936706048 dispatch_meta_bytes=458680 combine_payload_bytes=936706048 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=937164728 ingress_bytes=1465188352 link_bytes=1465188352 token_bytes=1873870776 weight_movement_bytes=528482304 total_nvlink_bytes=2402353080 weight_to_token_ratio=0.282027 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.204443171e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 0, 1, 3, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458752 combine_payload_bytes=939196416 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939655168 ingress_bytes=1820000256 link_bytes=1820000256 token_bytes=1878851584 weight_movement_bytes=880803840 total_nvlink_bytes=2759655424 weight_to_token_ratio=0.468799 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019124173e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=458776 combine_payload_bytes=939130880 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939589656 ingress_bytes=1467613184 link_bytes=1467613184 token_bytes=1878720536 weight_movement_bytes=528482304 total_nvlink_bytes=2407202840 weight_to_token_ratio=0.281299 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218021219e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129312 payload_rows_remote=112930 dup_rows=1760 dup_rows_remote=288 dedup_ratio=0.013428 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 8192, 1, 8191, 2, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0] payload_rows_per_dst=[16254, 16138, 16262, 16350, 15164, 16384, 16378, 16382] dup_rows_per_dst=[0, 4, 72, 6, 194, 12, 0, 1472] combine_rows_per_src=[16254, 16138, 16262, 16350, 15164, 16384, 16378, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458952 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=939983048 ingress_bytes=2348810240 link_bytes=2348810240 token_bytes=1879507144 weight_movement_bytes=1409286144 total_nvlink_bytes=3288793288 weight_to_token_ratio=0.749817 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315235690e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=458752 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939868160 ingress_bytes=1644052480 link_bytes=1644052480 token_bytes=1879277568 weight_movement_bytes=704643072 total_nvlink_bytes=2583920640 weight_to_token_ratio=0.374954 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206007627e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938491904 dispatch_meta_bytes=458744 combine_payload_bytes=938491904 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=938950648 ingress_bytes=1466974208 link_bytes=1466974208 token_bytes=1877442552 weight_movement_bytes=528482304 total_nvlink_bytes=2405924856 weight_to_token_ratio=0.281491 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.214443220e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114660 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000259 router_tokens_per_expert=[0, 8190, 0, 0, 0, 0, 0, 1, 8179, 8191, 8190, 1, 8191, 8191, 0, 2, 0, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 8189, 14, 1, 8191, 1] payload_rows_per_dst=[16380, 16384, 16360, 16384, 16384, 16384, 16378, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 2, 0, 30] combine_rows_per_src=[16380, 16384, 16360, 16384, 16384, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=925122560 dispatch_meta_bytes=452872 combine_payload_bytes=925122560 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=925575432 ingress_bytes=1629765632 link_bytes=1629765632 token_bytes=1850697992 weight_movement_bytes=704643072 total_nvlink_bytes=2555341064 weight_to_token_ratio=0.380744 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.126007242e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130894 payload_rows_remote=114510 dup_rows=178 dup_rows_remote=120 dedup_ratio=0.001358 router_tokens_per_expert=[8, 28, 8185, 0, 0, 0, 5, 0, 3, 1, 8181, 0, 0, 8185, 1, 8186, 0, 4, 8182, 8187, 0, 10, 0, 5, 0, 3, 5, 7, 8181, 8157, 0, 12] payload_rows_per_dst=[16384, 16372, 16370, 16374, 16382, 16380, 16294, 16338] dup_rows_per_dst=[58, 0, 0, 102, 0, 10, 8, 0] combine_rows_per_src=[16384, 16372, 16370, 16374, 16382, 16380, 16294, 16338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=458776 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939753496 ingress_bytes=1467777024 link_bytes=1467777024 token_bytes=1879048216 weight_movement_bytes=528482304 total_nvlink_bytes=2407530520 weight_to_token_ratio=0.281250 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218938654e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130980 payload_rows_remote=114596 dup_rows=92 dup_rows_remote=42 dedup_ratio=0.000702 router_tokens_per_expert=[0, 13, 8192, 0, 0, 0, 0, 25, 0, 0, 8192, 8, 0, 8192, 0, 8190, 3, 0, 8192, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16298, 16384, 16384, 16382] dup_rows_per_dst=[26, 50, 16, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16380, 16298, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131020 payload_rows_remote=114636 dup_rows=52 dup_rows_remote=8 dedup_ratio=0.000397 router_tokens_per_expert=[0, 8192, 3, 0, 0, 8182, 8, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 23, 0, 0, 0, 0, 8192, 0, 0, 1, 8192, 0, 8168, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16336] dup_rows_per_dst=[6, 0, 0, 0, 44, 0, 2, 0] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130914 payload_rows_remote=114530 dup_rows=158 dup_rows_remote=60 dedup_ratio=0.001205 router_tokens_per_expert=[0, 1, 8192, 0, 0, 0, 0, 29, 0, 0, 8192, 49, 0, 8192, 0, 8192, 0, 0, 8192, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16226, 16384, 16384, 16384] dup_rows_per_dst=[2, 58, 98, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16226, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938065920 dispatch_meta_bytes=458520 combine_payload_bytes=938065920 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=938524440 ingress_bytes=1642708992 link_bytes=1642708992 token_bytes=1876590360 weight_movement_bytes=704643072 total_nvlink_bytes=2581233432 weight_to_token_ratio=0.375491 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.198484655e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130954 payload_rows_remote=114580 dup_rows=118 dup_rows_remote=118 dedup_ratio=0.000900 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 39, 0, 2, 8187, 0, 0, 8191, 0, 8188, 31, 0, 8192, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8168, 0, 0] payload_rows_per_dst=[16384, 16384, 16378, 16374, 16330, 16384, 16384, 16336] dup_rows_per_dst=[0, 78, 0, 0, 40, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16378, 16374, 16330, 16384, 16384, 16336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129612 payload_rows_remote=113272 dup_rows=1460 dup_rows_remote=1438 dedup_ratio=0.011139 router_tokens_per_expert=[51, 0, 8157, 7, 0, 0, 76, 1, 0, 0, 8163, 1, 0, 8161, 5, 8165, 645, 24, 8140, 8156, 0, 1, 18, 22, 0, 31, 27, 10, 8135, 7495, 0, 45] payload_rows_per_dst=[16384, 16382, 16326, 16332, 16384, 16340, 16384, 15080] dup_rows_per_dst=[46, 42, 2, 0, 1266, 22, 82, 0] combine_rows_per_src=[16384, 16382, 16326, 16332, 16384, 16340, 16384, 15080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938770432 dispatch_meta_bytes=458552 combine_payload_bytes=938770432 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939228984 ingress_bytes=1291091968 link_bytes=1291091968 token_bytes=1877999416 weight_movement_bytes=352321536 total_nvlink_bytes=2230320952 weight_to_token_ratio=0.187605 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229576092e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126170 payload_rows_remote=109786 dup_rows=4902 dup_rows_remote=392 dedup_ratio=0.037399 router_tokens_per_expert=[6197, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8191, 1993, 0, 0, 8192] payload_rows_per_dst=[12394, 16228, 16170, 16384, 15860, 16366, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 4, 150, 0, 236, 4510] combine_rows_per_src=[12394, 16228, 16170, 16384, 15860, 16366, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939098112 dispatch_meta_bytes=458576 combine_payload_bytes=939098112 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939556688 ingress_bytes=1819901952 link_bytes=1819901952 token_bytes=1878654800 weight_movement_bytes=880803840 total_nvlink_bytes=2759458640 weight_to_token_ratio=0.468848 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019069127e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938229760 dispatch_meta_bytes=458360 combine_payload_bytes=938229760 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=938688120 ingress_bytes=1642872832 link_bytes=1642872832 token_bytes=1876917880 weight_movement_bytes=704643072 total_nvlink_bytes=2581560952 weight_to_token_ratio=0.375426 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.199402090e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130972 payload_rows_remote=114588 dup_rows=100 dup_rows_remote=94 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 8191, 0, 0, 0, 1, 9, 0, 0, 8191, 5, 2, 8191, 0, 8191, 33, 1, 8191, 8175, 0, 0, 2, 1, 2, 1, 1, 0, 8191, 8156, 1, 0] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16358, 16384, 16384, 16314] dup_rows_per_dst=[0, 18, 10, 2, 60, 4, 6, 0] combine_rows_per_src=[16382, 16384, 16382, 16384, 16358, 16384, 16384, 16314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938639360 dispatch_meta_bytes=458792 combine_payload_bytes=938639360 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939098152 ingress_bytes=1819443200 link_bytes=1819443200 token_bytes=1877737512 weight_movement_bytes=880803840 total_nvlink_bytes=2758541352 weight_to_token_ratio=0.469077 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018812245e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=927924224 dispatch_meta_bytes=458840 combine_payload_bytes=927924224 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=928383064 ingress_bytes=1456406528 link_bytes=1456406528 token_bytes=1856307288 weight_movement_bytes=528482304 total_nvlink_bytes=2384789592 weight_to_token_ratio=0.284695 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.155268623e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=899366912 dispatch_meta_bytes=440712 combine_payload_bytes=899366912 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=899807624 ingress_bytes=1956331520 link_bytes=1956331520 token_bytes=1799174536 weight_movement_bytes=1056964608 total_nvlink_bytes=2856139144 weight_to_token_ratio=0.587472 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.095463990e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119146 payload_rows_remote=102802 dup_rows=11926 dup_rows_remote=276 dedup_ratio=0.090988 router_tokens_per_expert=[7227, 9, 0, 5, 9, 2, 8135, 24, 7, 0, 0, 6, 8165, 4, 8175, 5, 0, 1, 0, 8181, 0, 6, 4, 0, 3, 8186, 0, 8185, 1012, 0, 0, 8185] payload_rows_per_dst=[16344, 16326, 16366, 16332, 16362, 16382, 16192, 4842] dup_rows_per_dst=[11650, 14, 18, 8, 184, 8, 4, 40] combine_rows_per_src=[16344, 16326, 16366, 16332, 16362, 16382, 16192, 4842] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938704896 dispatch_meta_bytes=458728 combine_payload_bytes=938704896 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939163624 ingress_bytes=1291026432 link_bytes=1291026432 token_bytes=1877868520 weight_movement_bytes=352321536 total_nvlink_bytes=2230190056 weight_to_token_ratio=0.187618 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229209118e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129250 payload_rows_remote=112866 dup_rows=1822 dup_rows_remote=1784 dedup_ratio=0.013901 router_tokens_per_expert=[7382, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 46, 0, 8192, 0, 8192, 0, 0, 61, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 703, 0, 0, 8192] payload_rows_per_dst=[15478, 16384, 16384, 16384, 16384, 16384, 16384, 15468] dup_rows_per_dst=[748, 38, 92, 252, 122, 0, 34, 536] combine_rows_per_src=[15478, 16384, 16384, 16384, 16384, 16384, 16384, 15468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131042 payload_rows_remote=114658 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000229 router_tokens_per_expert=[0, 8192, 0, 0, 0, 1, 0, 0, 8177, 8192, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16354, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 28] combine_rows_per_src=[16384, 16384, 16354, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129052 payload_rows_remote=112668 dup_rows=2020 dup_rows_remote=1934 dedup_ratio=0.015411 router_tokens_per_expert=[7200, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 43, 0, 8192, 0, 8192, 0, 0, 78, 0, 8192, 0, 0, 0, 1, 0, 8175, 0, 8191, 889, 0, 0, 8191] payload_rows_per_dst=[14400, 16384, 16384, 16384, 16384, 16384, 16350, 16382] dup_rows_per_dst=[0, 0, 86, 0, 156, 0, 0, 1778] combine_rows_per_src=[14400, 16384, 16384, 16384, 16384, 16384, 16350, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=842153984 dispatch_meta_bytes=412312 combine_payload_bytes=842153984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=842566296 ingress_bytes=1194475520 link_bytes=1194475520 token_bytes=1684720280 weight_movement_bytes=352321536 total_nvlink_bytes=2037041816 weight_to_token_ratio=0.209128 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.688564313e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127450 payload_rows_remote=111066 dup_rows=3622 dup_rows_remote=3622 dedup_ratio=0.027634 router_tokens_per_expert=[6385, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 23, 0, 8192, 0, 8192, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 2, 8192, 0, 8191, 1784, 1, 0, 8192] payload_rows_per_dst=[12770, 16378, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 46, 0, 2, 0, 4, 3570] combine_rows_per_src=[12770, 16378, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=924598272 dispatch_meta_bytes=458600 combine_payload_bytes=924598272 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=925056872 ingress_bytes=1629241344 link_bytes=1629241344 token_bytes=1849655144 weight_movement_bytes=704643072 total_nvlink_bytes=2554298216 weight_to_token_ratio=0.380959 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.123071448e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129822 payload_rows_remote=113442 dup_rows=1250 dup_rows_remote=1232 dedup_ratio=0.009537 router_tokens_per_expert=[7824, 34, 1, 23, 58, 0, 8153, 61, 29, 0, 0, 33, 8124, 24, 8046, 18, 0, 6, 11, 8112, 0, 27, 9, 0, 7, 8158, 2, 8163, 463, 6, 0, 8144] payload_rows_per_dst=[15694, 16372, 16314, 16140, 16246, 16380, 16332, 16344] dup_rows_per_dst=[70, 172, 58, 36, 12, 18, 2, 882] combine_rows_per_src=[15694, 16372, 16314, 16140, 16246, 16380, 16332, 16344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130124 payload_rows_remote=113740 dup_rows=948 dup_rows_remote=782 dedup_ratio=0.007233 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 8189, 0, 651, 7522, 0, 0, 0, 0, 8181, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 30] payload_rows_per_dst=[16360, 16262, 15986, 16044, 16342, 16362, 16384, 16384] dup_rows_per_dst=[0, 0, 6, 20, 0, 664, 92, 166] combine_rows_per_src=[16360, 16262, 15986, 16044, 16342, 16362, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130478 payload_rows_remote=114096 dup_rows=594 dup_rows_remote=594 dedup_ratio=0.004532 router_tokens_per_expert=[7899, 2, 0, 0, 0, 0, 8191, 4, 1, 0, 0, 1, 8190, 0, 8189, 1, 0, 1, 0, 8190, 2, 0, 0, 0, 3, 8188, 0, 8191, 292, 1, 0, 8190] payload_rows_per_dst=[15800, 16384, 16384, 16380, 16380, 16384, 16382, 16384] dup_rows_per_dst=[2, 6, 2, 0, 2, 0, 0, 582] combine_rows_per_src=[15800, 16384, 16384, 16380, 16380, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=458752 combine_payload_bytes=939278336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939737088 ingress_bytes=1291599872 link_bytes=1291599872 token_bytes=1879015424 weight_movement_bytes=352321536 total_nvlink_bytes=2231336960 weight_to_token_ratio=0.187503 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232420143e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=922976256 dispatch_meta_bytes=458408 combine_payload_bytes=922976256 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=923434664 ingress_bytes=1451458560 link_bytes=1451458560 token_bytes=1846410920 weight_movement_bytes=528482304 total_nvlink_bytes=2374893224 weight_to_token_ratio=0.286221 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.127562067e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=909852672 dispatch_meta_bytes=458752 combine_payload_bytes=909852672 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=910311424 ingress_bytes=1790656512 link_bytes=1790656512 token_bytes=1820164096 weight_movement_bytes=880803840 total_nvlink_bytes=2700967936 weight_to_token_ratio=0.483915 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.002692901e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=929316864 dispatch_meta_bytes=458696 combine_payload_bytes=929316864 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=929775560 ingress_bytes=1281638400 link_bytes=1281638400 token_bytes=1859092424 weight_movement_bytes=352321536 total_nvlink_bytes=2211413960 weight_to_token_ratio=0.189513 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.176640058e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=931758080 dispatch_meta_bytes=458088 combine_payload_bytes=931758080 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=932216168 ingress_bytes=2164883456 link_bytes=2164883456 token_bytes=1863974248 weight_movement_bytes=1233125376 total_nvlink_bytes=3097099624 weight_to_token_ratio=0.661557 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.212244369e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934674432 dispatch_meta_bytes=458760 combine_payload_bytes=934674432 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=935133192 ingress_bytes=1815478272 link_bytes=1815478272 token_bytes=1869807624 weight_movement_bytes=880803840 total_nvlink_bytes=2750611464 weight_to_token_ratio=0.471067 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.016592051e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130584 payload_rows_remote=114200 dup_rows=488 dup_rows_remote=472 dedup_ratio=0.003723 router_tokens_per_expert=[5, 0, 0, 7, 6, 0, 8185, 8, 5, 9, 0, 22, 8187, 6, 107, 8166, 0, 0, 0, 7, 8078, 0, 1, 8185, 0, 3, 4, 8182, 8179, 8184, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16348, 16382, 16160, 16370, 16172] dup_rows_per_dst=[16, 16, 194, 210, 0, 46, 6, 0] combine_rows_per_src=[16384, 16384, 16384, 16348, 16382, 16160, 16370, 16172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131016 payload_rows_remote=114632 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 8166, 0, 904, 7291, 0, 0, 0, 0, 8189, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 25] payload_rows_per_dst=[16384, 16384, 16334, 16384, 16384, 16378, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 0, 50] combine_rows_per_src=[16384, 16384, 16334, 16384, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=935526400 dispatch_meta_bytes=458688 combine_payload_bytes=935526400 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=935985088 ingress_bytes=1464008704 link_bytes=1464008704 token_bytes=1871511488 weight_movement_bytes=528482304 total_nvlink_bytes=2399993792 weight_to_token_ratio=0.282383 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.197837635e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131010 payload_rows_remote=114686 dup_rows=62 dup_rows_remote=62 dedup_ratio=0.000473 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8162, 0, 697, 7495, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8191, 8192, 8192, 0, 31] payload_rows_per_dst=[16384, 16384, 16324, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 62] combine_rows_per_src=[16384, 16384, 16324, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131042 payload_rows_remote=114658 dup_rows=30 dup_rows_remote=20 dedup_ratio=0.000229 router_tokens_per_expert=[1, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 4, 8185, 0, 555, 7642, 0, 0, 1, 0, 8187, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8190, 0, 5] payload_rows_per_dst=[16384, 16376, 16374, 16384, 16382, 16374, 16384, 16384] dup_rows_per_dst=[2, 0, 8, 10, 0, 0, 0, 10] combine_rows_per_src=[16384, 16376, 16374, 16384, 16382, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114688 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000107 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 8192, 2, 0, 8192, 0, 8192, 0, 0, 8192, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16370, 16384, 16384, 16384] dup_rows_per_dst=[0, 10, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16370, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939065344 dispatch_meta_bytes=458752 combine_payload_bytes=939065344 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939524096 ingress_bytes=1643708416 link_bytes=1643708416 token_bytes=1878589440 weight_movement_bytes=704643072 total_nvlink_bytes=2583232512 weight_to_token_ratio=0.375092 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204081012e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114672 dup_rows=16 dup_rows_remote=14 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 8191, 1, 0, 1, 0, 3, 8185, 0, 266, 7928, 1, 1, 1, 1, 8190, 1, 0, 8191, 0, 1, 1, 8191, 8190, 8191, 0, 0] payload_rows_per_dst=[16382, 16384, 16378, 16382, 16384, 16382, 16384, 16380] dup_rows_per_dst=[0, 2, 0, 6, 6, 0, 2, 0] combine_rows_per_src=[16382, 16384, 16378, 16382, 16384, 16382, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130412 payload_rows_remote=114462 dup_rows=660 dup_rows_remote=660 dedup_ratio=0.005035 router_tokens_per_expert=[15, 0, 0, 30, 36, 1, 8167, 44, 7, 38, 0, 145, 8150, 28, 188, 8033, 0, 0, 15, 41, 7975, 0, 0, 8162, 0, 6, 24, 8120, 8145, 8156, 1, 9] payload_rows_per_dst=[16364, 16384, 16380, 16370, 16384, 15950, 16288, 16292] dup_rows_per_dst=[30, 112, 320, 128, 40, 0, 12, 18] combine_rows_per_src=[16364, 16384, 16380, 16370, 16384, 15950, 16288, 16292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116884 payload_rows_remote=101522 dup_rows=14188 dup_rows_remote=1324 dedup_ratio=0.108246 router_tokens_per_expert=[0, 0, 491, 243, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 7458, 0, 8191, 0] payload_rows_per_dst=[3548, 16384, 16054, 16384, 16384, 16384, 16384, 15362] dup_rows_per_dst=[1020, 0, 0, 0, 48, 256, 0, 12864] combine_rows_per_src=[3548, 16384, 16054, 16384, 16384, 16384, 16384, 15362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 2, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458992 combine_payload_bytes=939507712 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939966704 ingress_bytes=1467990016 link_bytes=1467990016 token_bytes=1879474416 weight_movement_bytes=528482304 total_nvlink_bytes=2407956720 weight_to_token_ratio=0.281186 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220131320e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=458712 combine_payload_bytes=939278336 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=939737048 ingress_bytes=2172403712 link_bytes=2172403712 token_bytes=1879015384 weight_movement_bytes=1233125376 total_nvlink_bytes=3112140760 weight_to_token_ratio=0.656261 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216455398e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458808 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939982904 ingress_bytes=1996488704 link_bytes=1996488704 token_bytes=1879507000 weight_movement_bytes=1056964608 total_nvlink_bytes=2936471608 weight_to_token_ratio=0.562363 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117950337e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=458744 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939851768 ingress_bytes=1291714560 link_bytes=1291714560 token_bytes=1879244792 weight_movement_bytes=352321536 total_nvlink_bytes=2231566328 weight_to_token_ratio=0.187480 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233062348e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=937672704 dispatch_meta_bytes=460488 combine_payload_bytes=937672704 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=938133192 ingress_bytes=1642315776 link_bytes=1642315776 token_bytes=1875805896 weight_movement_bytes=704643072 total_nvlink_bytes=2580448968 weight_to_token_ratio=0.375648 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196282809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=831668224 dispatch_meta_bytes=411384 combine_payload_bytes=831668224 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=832079608 ingress_bytes=1712472064 link_bytes=1712472064 token_bytes=1663747832 weight_movement_bytes=880803840 total_nvlink_bytes=2544551672 weight_to_token_ratio=0.529409 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.589128737e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129704 payload_rows_remote=113328 dup_rows=1368 dup_rows_remote=116 dedup_ratio=0.010437 router_tokens_per_expert=[17, 0, 324, 289, 0, 2, 8182, 5, 4, 11, 8, 10, 1, 8186, 0, 6, 8175, 8185, 8187, 6, 8172, 0, 0, 1, 1, 0, 4, 10, 7566, 0, 8184, 0] payload_rows_per_dst=[16376, 16374, 16384, 16384, 16384, 16346, 16338, 15118] dup_rows_per_dst=[1252, 12, 52, 44, 4, 0, 4, 0] combine_rows_per_src=[16376, 16374, 16384, 16384, 16384, 16346, 16338, 15118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126512 payload_rows_remote=110128 dup_rows=4560 dup_rows_remote=4560 dedup_ratio=0.034790 router_tokens_per_expert=[0, 0, 1547, 733, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8190, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5912, 4, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16380, 16384, 16384, 11832] dup_rows_per_dst=[4560, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16380, 16380, 16384, 16384, 11832] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=928382976 dispatch_meta_bytes=453776 combine_payload_bytes=928382976 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928836752 ingress_bytes=1280704512 link_bytes=1280704512 token_bytes=1857219728 weight_movement_bytes=352321536 total_nvlink_bytes=2209541264 weight_to_token_ratio=0.189704 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.171410675e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=902168576 dispatch_meta_bytes=458752 combine_payload_bytes=902168576 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=902627328 ingress_bytes=1254490112 link_bytes=1254490112 token_bytes=1804795904 weight_movement_bytes=352321536 total_nvlink_bytes=2157117440 weight_to_token_ratio=0.195214 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.024620977e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130102 payload_rows_remote=113718 dup_rows=970 dup_rows_remote=970 dedup_ratio=0.007401 router_tokens_per_expert=[0, 0, 316, 190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8185, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7695, 22, 8171, 0] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16370, 16384, 16384, 15434] dup_rows_per_dst=[970, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16378, 16370, 16384, 16384, 15434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130922 payload_rows_remote=114546 dup_rows=150 dup_rows_remote=150 dedup_ratio=0.001144 router_tokens_per_expert=[0, 1, 66, 7, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 8191, 8192, 8191, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 8119, 4, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16382, 16384, 16382, 16246] dup_rows_per_dst=[148, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16376, 16382, 16384, 16382, 16246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130410 payload_rows_remote=114026 dup_rows=662 dup_rows_remote=602 dedup_ratio=0.005051 router_tokens_per_expert=[7861, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 30, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 301, 0, 0, 8192] payload_rows_per_dst=[15722, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 60, 0, 0, 602] combine_rows_per_src=[15722, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130040 payload_rows_remote=113656 dup_rows=1032 dup_rows_remote=1026 dedup_ratio=0.007874 router_tokens_per_expert=[0, 0, 349, 158, 1, 1, 8188, 0, 0, 4, 1, 1, 0, 8190, 0, 0, 8191, 8191, 8191, 2, 8190, 0, 0, 0, 1, 0, 2, 1, 7684, 0, 8190, 0] payload_rows_per_dst=[16382, 16378, 16384, 16380, 16384, 16380, 16384, 15368] dup_rows_per_dst=[1012, 2, 10, 0, 2, 0, 6, 0] combine_rows_per_src=[16382, 16378, 16384, 16380, 16384, 16380, 16384, 15368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129378 payload_rows_remote=112994 dup_rows=1694 dup_rows_remote=196 dedup_ratio=0.012924 router_tokens_per_expert=[0, 0, 8190, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16244, 16112, 16384, 16384, 16384, 16380, 15106, 16384] dup_rows_per_dst=[0, 0, 136, 0, 52, 0, 8, 1498] combine_rows_per_src=[16244, 16112, 16384, 16384, 16384, 16380, 15106, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127658 payload_rows_remote=111490 dup_rows=3414 dup_rows_remote=3414 dedup_ratio=0.026047 router_tokens_per_expert=[13, 0, 151, 1379, 0, 10, 8156, 27, 16, 81, 36, 34, 0, 8158, 1, 23, 8148, 8159, 8171, 33, 8084, 0, 0, 0, 0, 0, 30, 10, 6646, 0, 8165, 5] payload_rows_per_dst=[16330, 16366, 16384, 16362, 16362, 16168, 16384, 13302] dup_rows_per_dst=[3086, 20, 268, 2, 0, 0, 38, 0] combine_rows_per_src=[16330, 16366, 16384, 16362, 16362, 16168, 16384, 13302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=931577856 dispatch_meta_bytes=458752 combine_payload_bytes=931577856 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=932036608 ingress_bytes=1283899392 link_bytes=1283899392 token_bytes=1863614464 weight_movement_bytes=352321536 total_nvlink_bytes=2215936000 weight_to_token_ratio=0.189053 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.189300669e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938360832 dispatch_meta_bytes=458784 combine_payload_bytes=938360832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938819616 ingress_bytes=1290682368 link_bytes=1290682368 token_bytes=1877180448 weight_movement_bytes=352321536 total_nvlink_bytes=2229501984 weight_to_token_ratio=0.187687 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227282503e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934100992 dispatch_meta_bytes=458512 combine_payload_bytes=934100992 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=934559504 ingress_bytes=1462583296 link_bytes=1462583296 token_bytes=1868660496 weight_movement_bytes=528482304 total_nvlink_bytes=2397142800 weight_to_token_ratio=0.282813 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.189855945e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=931069952 dispatch_meta_bytes=458728 combine_payload_bytes=931069952 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=931528680 ingress_bytes=1283391488 link_bytes=1283391488 token_bytes=1862598632 weight_movement_bytes=352321536 total_nvlink_bytes=2214920168 weight_to_token_ratio=0.189156 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.186456619e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=925646848 dispatch_meta_bytes=452760 combine_payload_bytes=925646848 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=926099608 ingress_bytes=2158772224 link_bytes=2158772224 token_bytes=1851746456 weight_movement_bytes=1233125376 total_nvlink_bytes=3084871832 weight_to_token_ratio=0.665926 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.208822334e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129544 payload_rows_remote=113174 dup_rows=1528 dup_rows_remote=1516 dedup_ratio=0.011658 router_tokens_per_expert=[0, 0, 8185, 0, 8185, 10, 9, 8184, 0, 8178, 0, 0, 8, 0, 7, 8, 8182, 5, 0, 0, 8116, 6, 23, 0, 0, 0, 75, 5, 8159, 7, 8184, 0] payload_rows_per_dst=[16370, 16382, 16356, 16198, 16374, 16368, 16384, 15112] dup_rows_per_dst=[12, 200, 8, 30, 4, 1130, 144, 0] combine_rows_per_src=[16370, 16382, 16356, 16198, 16374, 16368, 16384, 15112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131020 payload_rows_remote=114636 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8191, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 25, 0, 0, 8168, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16336] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 50, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=913326080 dispatch_meta_bytes=459616 combine_payload_bytes=913326080 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=913785696 ingress_bytes=1265647616 link_bytes=1265647616 token_bytes=1827111776 weight_movement_bytes=352321536 total_nvlink_bytes=2179433312 weight_to_token_ratio=0.192830 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.087098342e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27678983328 prefetch_bytes=4404019200 reduce_bytes=8808038400 weight_movement_bytes=13212057600 egress_bytes=13843023008 ingress_bytes=27048017920 link_bytes=27048017920 total_nvlink_bytes=40891040928 weight_to_token_ratio=0.477332 prefetch_share_of_ingress=0.162822 dedup_ratio=0.071687 prefetch_slots=50 unused_prefetch_slots=206 slot_utilization=0.195312 reduce_slots=50 remote_demand=25 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019768 gemm_inner_pad_upper_bound=20701 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.466100622e-05 prefetch_bytes_per_gemm_flop=1.541291824e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=927121408 dispatch_meta_bytes=458760 combine_payload_bytes=927121408 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=927580168 ingress_bytes=1455603712 link_bytes=1455603712 token_bytes=1854701576 weight_movement_bytes=528482304 total_nvlink_bytes=2383183880 weight_to_token_ratio=0.284942 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.150773188e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939098112 dispatch_meta_bytes=458752 combine_payload_bytes=939098112 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=939556864 ingress_bytes=1996062720 link_bytes=1996062720 token_bytes=1878654976 weight_movement_bytes=1056964608 total_nvlink_bytes=2935619584 weight_to_token_ratio=0.562618 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117711803e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114686 dup_rows=6 dup_rows_remote=4 dedup_ratio=0.000046 router_tokens_per_expert=[0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8186, 5, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 4, 0] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114686 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000046 router_tokens_per_expert=[0, 0, 8191, 0, 8190, 0, 0, 8192, 0, 8192, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16382, 16384, 16384, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 0, 0, 4, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131026 payload_rows_remote=114642 dup_rows=46 dup_rows_remote=6 dedup_ratio=0.000351 router_tokens_per_expert=[0, 0, 8191, 0, 8191, 1, 1, 8191, 0, 8185, 0, 0, 1, 0, 1, 1, 8191, 0, 0, 1, 8191, 0, 0, 0, 0, 11, 10, 1, 8174, 1, 8190, 3] payload_rows_per_dst=[16382, 16384, 16370, 16384, 16384, 16382, 16384, 16356] dup_rows_per_dst=[0, 2, 0, 4, 0, 0, 40, 0] combine_rows_per_src=[16382, 16384, 16370, 16384, 16384, 16382, 16384, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114674 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000107 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 84, 8114, 0, 0, 0, 0, 8186, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 1] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16372, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 12, 0, 0, 0, 2] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28058397160 prefetch_bytes=5108662272 reduce_bytes=4932501504 weight_movement_bytes=10041163776 egress_bytes=14032792040 ingress_bytes=24066768896 link_bytes=24066768896 total_nvlink_bytes=38099560936 weight_to_token_ratio=0.357867 prefetch_share_of_ingress=0.212270 dedup_ratio=0.058741 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=28 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020005 gemm_inner_pad_upper_bound=20574 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.422741241e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28441117944 prefetch_bytes=4932501504 reduce_bytes=5460983808 weight_movement_bytes=10393485312 egress_bytes=14224196856 ingress_bytes=24610406400 link_bytes=24610406400 total_nvlink_bytes=38834603256 weight_to_token_ratio=0.365439 prefetch_share_of_ingress=0.200423 dedup_ratio=0.049630 prefetch_slots=56 unused_prefetch_slots=200 slot_utilization=0.218750 reduce_slots=31 remote_demand=28 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019176 gemm_inner_pad_upper_bound=21082 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.613000184e-05 prefetch_bytes_per_gemm_flop=1.726246843e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129602 payload_rows_remote=114328 dup_rows=1470 dup_rows_remote=1470 dedup_ratio=0.011215 router_tokens_per_expert=[22, 71, 8156, 16, 8162, 43, 31, 8154, 0, 8137, 3, 0, 30, 3, 35, 40, 8143, 28, 0, 0, 7630, 0, 7, 0, 0, 18, 531, 1, 8068, 36, 8157, 14] payload_rows_per_dst=[16378, 16384, 16280, 16384, 16342, 15274, 16324, 16236] dup_rows_per_dst=[152, 72, 0, 156, 0, 0, 1090, 0] combine_rows_per_src=[16378, 16384, 16280, 16384, 16342, 15274, 16324, 16236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458760 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966472 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474184 weight_movement_bytes=352321536 total_nvlink_bytes=2231795720 weight_to_token_ratio=0.187458 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458768 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966480 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474192 weight_movement_bytes=352321536 total_nvlink_bytes=2231795728 weight_to_token_ratio=0.187458 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=458592 combine_payload_bytes=939147264 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939605856 ingress_bytes=1291468800 link_bytes=1291468800 token_bytes=1878753120 weight_movement_bytes=352321536 total_nvlink_bytes=2231074656 weight_to_token_ratio=0.187529 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231686194e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=458752 combine_payload_bytes=939409408 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939868160 ingress_bytes=1291730944 link_bytes=1291730944 token_bytes=1879277568 weight_movement_bytes=352321536 total_nvlink_bytes=2231599104 weight_to_token_ratio=0.187477 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233154091e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=936574976 dispatch_meta_bytes=463192 combine_payload_bytes=936574976 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937038168 ingress_bytes=1288896512 link_bytes=1288896512 token_bytes=1873613144 weight_movement_bytes=352321536 total_nvlink_bytes=2225934680 weight_to_token_ratio=0.188044 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.217282455e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29015179848 prefetch_bytes=4932501504 reduce_bytes=4404019200 weight_movement_bytes=9336520704 egress_bytes=14511276616 ingress_bytes=23840423936 link_bytes=23840423936 total_nvlink_bytes=38351700552 weight_to_token_ratio=0.321781 prefetch_share_of_ingress=0.206897 dedup_ratio=0.036351 prefetch_slots=56 unused_prefetch_slots=200 slot_utilization=0.218750 reduce_slots=25 remote_demand=28 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019176 gemm_inner_pad_upper_bound=21209 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.343526410e-05 prefetch_bytes_per_gemm_flop=1.726246843e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28793258728 prefetch_bytes=5813305344 reduce_bytes=2466250752 weight_movement_bytes=8279556096 egress_bytes=14400291560 ingress_bytes=22672523264 link_bytes=22672523264 total_nvlink_bytes=37072814824 weight_to_token_ratio=0.287552 prefetch_share_of_ingress=0.256403 dedup_ratio=0.043751 prefetch_slots=66 unused_prefetch_slots=190 slot_utilization=0.257812 reduce_slots=14 remote_demand=33 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=20955 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.934791644e-05 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28702207888 prefetch_bytes=4580179968 reduce_bytes=3170893824 weight_movement_bytes=7751073792 egress_bytes=14354755472 ingress_bytes=22098526208 link_bytes=22098526208 total_nvlink_bytes=36453281680 weight_to_token_ratio=0.270051 prefetch_share_of_ingress=0.207262 dedup_ratio=0.041642 prefetch_slots=52 unused_prefetch_slots=204 slot_utilization=0.203125 reduce_slots=18 remote_demand=26 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019531 gemm_inner_pad_upper_bound=20320 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.733907649e-05 prefetch_bytes_per_gemm_flop=1.602943497e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29148981088 prefetch_bytes=5813305344 reduce_bytes=2818572288 weight_movement_bytes=8631877632 egress_bytes=14578181984 ingress_bytes=23202676736 link_bytes=23202676736 total_nvlink_bytes=37780858720 weight_to_token_ratio=0.296130 prefetch_share_of_ingress=0.250545 dedup_ratio=0.034727 prefetch_slots=66 unused_prefetch_slots=190 slot_utilization=0.257812 reduce_slots=16 remote_demand=33 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019650 gemm_inner_pad_upper_bound=20701 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.120331528e-05 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130220 payload_rows_remote=113848 dup_rows=852 dup_rows_remote=852 dedup_ratio=0.006500 router_tokens_per_expert=[0, 0, 120, 306, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8186, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7773, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16372, 16384, 16384, 15546] dup_rows_per_dst=[852, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16382, 16372, 16384, 16384, 15546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=11 export_slots=11 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 1, 0, 2, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=932642816 dispatch_meta_bytes=458800 combine_payload_bytes=932642816 prefetch_bytes=0 reduce_bytes=1937768448 egress_bytes=933101616 ingress_bytes=2870411264 link_bytes=2870411264 token_bytes=1865744432 weight_movement_bytes=1937768448 total_nvlink_bytes=3803512880 weight_to_token_ratio=1.038603 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.607310491e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966464 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474176 weight_movement_bytes=352321536 total_nvlink_bytes=2231795712 weight_to_token_ratio=0.187458 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28942315632 prefetch_bytes=4580179968 reduce_bytes=8103395328 weight_movement_bytes=12683575296 egress_bytes=14474817648 ingress_bytes=27151073280 link_bytes=27151073280 total_nvlink_bytes=41625890928 weight_to_token_ratio=0.438236 prefetch_share_of_ingress=0.168692 dedup_ratio=0.034390 prefetch_slots=52 unused_prefetch_slots=204 slot_utilization=0.203125 reduce_slots=46 remote_demand=26 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019058 gemm_inner_pad_upper_bound=21082 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.502167310e-05 prefetch_bytes_per_gemm_flop=1.602943497e-05 anomalies=none - [2026-09-16 21:37:05] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 575.7 | throughput per GPU (TFLOP/s/GPU): 579.1 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.039263E+01 | loss scale: 1.0 | grad norm: 16.268 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94920 payload_rows_remote=79998 dup_rows=36152 dup_rows_remote=28008 dedup_ratio=0.275818 router_tokens_per_expert=[1, 3187, 2341, 5, 1705, 0, 2636, 0, 1666, 2782, 149, 3364, 3824, 2799, 0, 5942, 5943, 5466, 1012, 9, 229, 4536, 2320, 1, 0, 2332, 433, 2475, 4245, 27, 5999, 108] payload_rows_per_dst=[14922, 14400, 14456, 9722, 13890, 11692, 8948, 6890] dup_rows_per_dst=[8144, 6438, 9114, 1408, 6948, 2480, 1532, 88] combine_rows_per_src=[14922, 14400, 14456, 9722, 13890, 11692, 8948, 6890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95984 payload_rows_remote=81660 dup_rows=35088 dup_rows_remote=28586 dedup_ratio=0.267700 router_tokens_per_expert=[5, 3725, 2823, 2, 2212, 3, 2168, 19, 1916, 2087, 123, 3359, 3585, 3167, 10, 5459, 6330, 4518, 1514, 51, 212, 3815, 2187, 2, 3, 1981, 533, 3046, 4789, 49, 5710, 133] payload_rows_per_dst=[15068, 14324, 12718, 11426, 12744, 10426, 9508, 9770] dup_rows_per_dst=[9462, 6502, 6012, 3702, 5614, 2006, 1618, 172] combine_rows_per_src=[15068, 14324, 12718, 11426, 12744, 10426, 9508, 9770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=92706 payload_rows_remote=77898 dup_rows=38366 dup_rows_remote=30360 dedup_ratio=0.292709 router_tokens_per_expert=[8, 3992, 2448, 16, 2179, 4, 1911, 13, 1994, 1933, 89, 3276, 3971, 2266, 11, 5539, 6038, 4571, 1566, 7, 532, 5131, 2634, 5, 1, 1785, 517, 2518, 3970, 59, 6374, 178] payload_rows_per_dst=[10266, 7112, 11222, 12578, 7912, 14594, 14214, 14808] dup_rows_per_dst=[2662, 1102, 3362, 5348, 1730, 9500, 6656, 8006] combine_rows_per_src=[10266, 7112, 11222, 12578, 7912, 14594, 14214, 14808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97898 payload_rows_remote=85150 dup_rows=33174 dup_rows_remote=28718 dedup_ratio=0.253098 router_tokens_per_expert=[7, 4279, 2995, 3, 1888, 0, 3124, 4, 1490, 2112, 168, 3454, 3340, 3082, 3, 4903, 5322, 5707, 998, 22, 207, 3685, 2945, 1, 6, 2024, 402, 2976, 4553, 54, 5707, 75] payload_rows_per_dst=[11322, 14092, 11174, 12748, 13320, 11216, 9598, 14428] dup_rows_per_dst=[3246, 6526, 3274, 4456, 4892, 2460, 1970, 6350] combine_rows_per_src=[11322, 14092, 11174, 12748, 13320, 11216, 9598, 14428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96274 payload_rows_remote=84694 dup_rows=34798 dup_rows_remote=31664 dedup_ratio=0.265488 router_tokens_per_expert=[6, 2986, 2986, 8, 2019, 4, 2580, 9, 1998, 1477, 156, 3726, 3963, 2651, 3, 5723, 5726, 4792, 1609, 35, 219, 4375, 2389, 12, 11, 2837, 552, 2555, 3934, 59, 5846, 290] payload_rows_per_dst=[13272, 14364, 11580, 13426, 12648, 11452, 10032, 9500] dup_rows_per_dst=[6320, 6482, 3134, 6720, 5952, 2538, 1878, 1774] combine_rows_per_src=[13272, 14364, 11580, 13426, 12648, 11452, 10032, 9500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96088 payload_rows_remote=83304 dup_rows=34984 dup_rows_remote=31316 dedup_ratio=0.266907 router_tokens_per_expert=[2, 3946, 2381, 2, 2262, 0, 2499, 0, 1189, 2628, 113, 3615, 3162, 3275, 0, 5667, 6450, 4574, 1059, 7, 239, 5017, 2960, 10, 0, 2047, 452, 2934, 3455, 32, 5520, 39] payload_rows_per_dst=[10190, 8760, 11576, 12488, 12354, 12784, 14416, 13520] dup_rows_per_dst=[2472, 2418, 3514, 4872, 5558, 3668, 7910, 4572] combine_rows_per_src=[10190, 8760, 11576, 12488, 12354, 12784, 14416, 13520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=93636 payload_rows_remote=79802 dup_rows=37436 dup_rows_remote=29816 dedup_ratio=0.285614 router_tokens_per_expert=[4, 3775, 2185, 0, 2143, 0, 2725, 1, 1923, 2691, 192, 2898, 4444, 3049, 1, 5828, 5420, 4597, 731, 3, 266, 4677, 2728, 13, 0, 2133, 560, 2745, 3501, 43, 6154, 106] payload_rows_per_dst=[9722, 8338, 11596, 13578, 8334, 13986, 13834, 14248] dup_rows_per_dst=[2206, 1400, 3812, 7438, 2350, 7250, 7620, 5360] combine_rows_per_src=[9722, 8338, 11596, 13578, 8334, 13986, 13834, 14248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 2, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=655343616 dispatch_meta_bytes=432024 combine_payload_bytes=655343616 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=655775640 ingress_bytes=1007665152 link_bytes=1007665152 token_bytes=1311119256 weight_movement_bytes=352321536 total_nvlink_bytes=1663440792 weight_to_token_ratio=0.268718 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=15616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.642504231e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=668958720 dispatch_meta_bytes=440984 combine_payload_bytes=668958720 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=669399704 ingress_bytes=1021280256 link_bytes=1021280256 token_bytes=1338358424 weight_movement_bytes=352321536 total_nvlink_bytes=1690679960 weight_to_token_ratio=0.263249 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=14592 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.718743130e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=638140416 dispatch_meta_bytes=433032 combine_payload_bytes=638140416 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=638573448 ingress_bytes=1518944256 link_bytes=1518944256 token_bytes=1276713864 weight_movement_bytes=880803840 total_nvlink_bytes=2157517704 weight_to_token_ratio=0.689899 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=16256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.505453795e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=697548800 dispatch_meta_bytes=455472 combine_payload_bytes=697548800 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=698004272 ingress_bytes=1754513408 link_bytes=1754513408 token_bytes=1395553072 weight_movement_bytes=1056964608 total_nvlink_bytes=2452517680 weight_to_token_ratio=0.757380 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=12928 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.824542715e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=693813248 dispatch_meta_bytes=465432 combine_payload_bytes=693813248 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=694278680 ingress_bytes=1046134784 link_bytes=1046134784 token_bytes=1388091928 weight_movement_bytes=352321536 total_nvlink_bytes=1740413464 weight_to_token_ratio=0.253817 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=13696 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.857918112e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=682426368 dispatch_meta_bytes=458480 combine_payload_bytes=682426368 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=682884848 ingress_bytes=1034747904 link_bytes=1034747904 token_bytes=1365311216 weight_movement_bytes=352321536 total_nvlink_bytes=1717632752 weight_to_token_ratio=0.258052 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=18688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.794156337e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=653737984 dispatch_meta_bytes=438472 combine_payload_bytes=653737984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=654176456 ingress_bytes=1006059520 link_bytes=1006059520 token_bytes=1307914440 weight_movement_bytes=352321536 total_nvlink_bytes=1660235976 weight_to_token_ratio=0.269377 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=11776 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.633513362e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96084 payload_rows_remote=83794 dup_rows=34988 dup_rows_remote=30452 dedup_ratio=0.266937 router_tokens_per_expert=[11, 3666, 2216, 6, 1940, 2, 2862, 1, 1685, 2022, 179, 3378, 3002, 2574, 5, 5504, 6113, 4413, 1678, 8, 163, 4129, 3207, 11, 1, 2085, 523, 3359, 4547, 45, 6056, 145] payload_rows_per_dst=[9576, 13144, 11154, 9524, 12290, 11818, 13968, 14610] dup_rows_per_dst=[2222, 5670, 3374, 1638, 4536, 3202, 7370, 6976] combine_rows_per_src=[9576, 13144, 11154, 9524, 12290, 11818, 13968, 14610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128010 payload_rows_remote=111662 dup_rows=3062 dup_rows_remote=3062 dedup_ratio=0.023361 router_tokens_per_expert=[182, 9, 7676, 6934, 0, 5, 0, 8169, 70, 1246, 0, 2, 1, 7, 8104, 4, 69, 0, 8164, 0, 2, 0, 8, 8186, 0, 7983, 29, 1, 4, 0, 8187, 494] payload_rows_per_dst=[14054, 16348, 16314, 16218, 16344, 16384, 15970, 16378] dup_rows_per_dst=[196, 0, 1674, 14, 122, 8, 56, 992] combine_rows_per_src=[14054, 16348, 16314, 16218, 16344, 16384, 15970, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114582 payload_rows_remote=98532 dup_rows=16490 dup_rows_remote=1524 dedup_ratio=0.125809 router_tokens_per_expert=[164, 2, 8052, 7332, 1, 3, 0, 8128, 90, 750, 0, 0, 7, 1, 8179, 0, 7, 1, 8171, 3, 4, 0, 1, 8186, 0, 8168, 36, 12, 2, 0, 8179, 57] payload_rows_per_dst=[16050, 16268, 9032, 16372, 16354, 16382, 15872, 8252] dup_rows_per_dst=[14966, 594, 364, 298, 96, 10, 128, 34] combine_rows_per_src=[16050, 16268, 9032, 16372, 16354, 16382, 15872, 8252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129436 payload_rows_remote=113162 dup_rows=1636 dup_rows_remote=1628 dedup_ratio=0.012482 router_tokens_per_expert=[289, 0, 7739, 7370, 2, 0, 7, 8184, 20, 855, 0, 0, 0, 10, 8128, 3, 37, 0, 8136, 0, 6, 0, 0, 8183, 0, 8173, 40, 11, 0, 0, 8157, 186] payload_rows_per_dst=[15146, 16382, 16246, 16274, 16332, 16366, 16348, 16342] dup_rows_per_dst=[172, 4, 982, 8, 14, 12, 100, 344] combine_rows_per_src=[15146, 16382, 16246, 16274, 16332, 16366, 16348, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125226 payload_rows_remote=108948 dup_rows=5846 dup_rows_remote=4654 dedup_ratio=0.044601 router_tokens_per_expert=[124, 0, 8062, 5462, 2, 0, 2, 8088, 53, 620, 0, 0, 0, 8, 8189, 0, 90, 0, 8122, 2, 4, 1, 2, 8184, 0, 8130, 65, 90, 3, 7, 8135, 2091] payload_rows_per_dst=[11078, 16180, 16278, 16380, 16346, 16376, 16308, 16280] dup_rows_per_dst=[94, 4, 1192, 14, 82, 6, 262, 4192] combine_rows_per_src=[11078, 16180, 16278, 16380, 16346, 16376, 16308, 16280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125584 payload_rows_remote=109234 dup_rows=5488 dup_rows_remote=3386 dedup_ratio=0.041870 router_tokens_per_expert=[46, 7, 7576, 5862, 13, 0, 1, 8158, 79, 1670, 0, 0, 6, 18, 8178, 4, 5, 0, 8149, 3, 0, 4, 5, 8177, 1, 8183, 233, 8, 11, 5, 8170, 964] payload_rows_per_dst=[11820, 16318, 15830, 16370, 16152, 16370, 16374, 16350] dup_rows_per_dst=[10, 26, 2820, 42, 10, 2, 476, 2102] combine_rows_per_src=[11820, 16318, 15830, 16370, 16152, 16370, 16374, 16350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129500 payload_rows_remote=113116 dup_rows=1572 dup_rows_remote=1572 dedup_ratio=0.011993 router_tokens_per_expert=[205, 0, 7938, 7571, 1, 0, 0, 8150, 7, 439, 0, 0, 0, 0, 8178, 1, 6, 0, 8185, 0, 0, 0, 0, 8192, 0, 8179, 31, 19, 0, 0, 8162, 272] payload_rows_per_dst=[15240, 16300, 16164, 16358, 16370, 16384, 16360, 16324] dup_rows_per_dst=[312, 2, 604, 0, 12, 0, 98, 544] combine_rows_per_src=[15240, 16300, 16164, 16358, 16370, 16384, 16360, 16324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=686440448 dispatch_meta_bytes=456984 combine_payload_bytes=686440448 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=686897432 ingress_bytes=1919565824 link_bytes=1919565824 token_bytes=1373337880 weight_movement_bytes=1233125376 total_nvlink_bytes=2606463256 weight_to_token_ratio=0.897904 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=15872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.074876735e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=914735104 dispatch_meta_bytes=458896 combine_payload_bytes=914735104 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=915194000 ingress_bytes=1267056640 link_bytes=1267056640 token_bytes=1829929104 weight_movement_bytes=352321536 total_nvlink_bytes=2182250640 weight_to_token_ratio=0.192533 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.094988289e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130224 payload_rows_remote=113868 dup_rows=848 dup_rows_remote=794 dedup_ratio=0.006470 router_tokens_per_expert=[233, 0, 7939, 7849, 3, 0, 0, 8189, 6, 250, 0, 0, 0, 0, 8192, 0, 16, 0, 8182, 0, 0, 0, 0, 8192, 0, 8178, 8, 19, 0, 0, 8188, 92] payload_rows_per_dst=[15878, 16382, 16096, 16384, 16368, 16384, 16356, 16376] dup_rows_per_dst=[286, 2, 294, 0, 28, 0, 54, 184] combine_rows_per_src=[15878, 16382, 16096, 16384, 16368, 16384, 16356, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=927023104 dispatch_meta_bytes=459160 combine_payload_bytes=927023104 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=927482264 ingress_bytes=1279344640 link_bytes=1279344640 token_bytes=1854505368 weight_movement_bytes=352321536 total_nvlink_bytes=2206826904 weight_to_token_ratio=0.189981 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.163795959e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=807174144 dispatch_meta_bytes=400224 combine_payload_bytes=807174144 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=807574368 ingress_bytes=1511817216 link_bytes=1511817216 token_bytes=1614748512 weight_movement_bytes=704643072 total_nvlink_bytes=2319391584 weight_to_token_ratio=0.436379 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=29312 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.465545346e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=892502016 dispatch_meta_bytes=454408 combine_payload_bytes=892502016 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=892956424 ingress_bytes=1244823552 link_bytes=1244823552 token_bytes=1785458440 weight_movement_bytes=352321536 total_nvlink_bytes=2137779976 weight_to_token_ratio=0.197328 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.024621 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.970492276e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=894844928 dispatch_meta_bytes=450480 combine_payload_bytes=894844928 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=895295408 ingress_bytes=2127970304 link_bytes=2127970304 token_bytes=1790140336 weight_movement_bytes=1233125376 total_nvlink_bytes=3023265712 weight_to_token_ratio=0.688843 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.191574544e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=926646272 dispatch_meta_bytes=458752 combine_payload_bytes=926646272 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=927105024 ingress_bytes=1102807040 link_bytes=1102807040 token_bytes=1853751296 weight_movement_bytes=176160768 total_nvlink_bytes=2029912064 weight_to_token_ratio=0.095029 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.175259090e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=932806656 dispatch_meta_bytes=458648 combine_payload_bytes=932806656 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=933265304 ingress_bytes=1813610496 link_bytes=1813610496 token_bytes=1866071960 weight_movement_bytes=880803840 total_nvlink_bytes=2746875800 weight_to_token_ratio=0.472010 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.015546174e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123074 payload_rows_remote=106692 dup_rows=7998 dup_rows_remote=7990 dedup_ratio=0.061020 router_tokens_per_expert=[8184, 3, 2, 0, 0, 0, 4, 8191, 0, 297, 5316, 8188, 0, 0, 0, 6911, 1, 0, 0, 0, 680, 8147, 0, 0, 2, 0, 3224, 0, 8190, 14, 8181, 1] payload_rows_per_dst=[16368, 16382, 16042, 14630, 16382, 13632, 13256, 16382] dup_rows_per_dst=[10, 8, 4184, 2362, 2, 852, 572, 8] combine_rows_per_src=[16368, 16382, 16042, 14630, 16382, 13632, 13256, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130024 payload_rows_remote=113640 dup_rows=1048 dup_rows_remote=990 dedup_ratio=0.007996 router_tokens_per_expert=[187, 1, 7852, 7697, 0, 0, 1, 8130, 92, 647, 0, 0, 0, 0, 8166, 4, 29, 0, 8192, 0, 3, 3, 0, 8188, 0, 8147, 6, 2, 1, 0, 8168, 20] payload_rows_per_dst=[15678, 16262, 16336, 16334, 16384, 16380, 16304, 16346] dup_rows_per_dst=[92, 0, 846, 6, 58, 8, 6, 32] combine_rows_per_src=[15678, 16262, 16336, 16334, 16384, 16380, 16304, 16346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120188 payload_rows_remote=106254 dup_rows=10884 dup_rows_remote=7684 dedup_ratio=0.083038 router_tokens_per_expert=[8191, 14, 7, 1, 0, 1, 23, 8191, 0, 42, 4922, 8188, 0, 0, 0, 6333, 0, 1, 1, 0, 2395, 8184, 2, 0, 0, 1, 2659, 0, 8190, 0, 8188, 2] payload_rows_per_dst=[16384, 16384, 13934, 12666, 16382, 16372, 11686, 16380] dup_rows_per_dst=[42, 46, 3200, 0, 2, 4790, 2804, 0] combine_rows_per_src=[16384, 16384, 13934, 12666, 16382, 16372, 11686, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127698 payload_rows_remote=112188 dup_rows=3374 dup_rows_remote=3372 dedup_ratio=0.025742 router_tokens_per_expert=[8185, 14, 3, 0, 1, 0, 2, 8191, 0, 115, 6863, 8188, 0, 1, 0, 7755, 0, 0, 0, 0, 280, 8154, 0, 0, 0, 0, 1403, 0, 8191, 0, 8190, 0] payload_rows_per_dst=[16370, 16384, 13954, 15510, 16384, 16334, 16384, 16378] dup_rows_per_dst=[34, 4, 2, 2, 0, 534, 2798, 0] combine_rows_per_src=[16370, 16384, 13954, 15510, 16384, 16334, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=103784 payload_rows_remote=87400 dup_rows=27288 dup_rows_remote=27262 dedup_ratio=0.208191 router_tokens_per_expert=[8189, 4, 1, 1, 19, 0, 0, 8187, 0, 36, 7163, 8179, 0, 0, 1, 7881, 0, 4, 4, 2, 383, 8177, 4, 0, 0, 1, 916, 0, 8188, 1, 8192, 3] payload_rows_per_dst=[16384, 16382, 16378, 16182, 16384, 3888, 1834, 16352] dup_rows_per_dst=[26, 32, 14372, 12772, 34, 50, 0, 2] combine_rows_per_src=[16384, 16382, 16378, 16182, 16384, 3888, 1834, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119924 payload_rows_remote=103564 dup_rows=11148 dup_rows_remote=11080 dedup_ratio=0.085052 router_tokens_per_expert=[8179, 0, 0, 1, 22, 4, 3, 8185, 6, 84, 7085, 8175, 12, 1, 2, 6360, 3, 1, 2, 1, 384, 8182, 0, 0, 10, 5, 2442, 20, 8178, 2, 8186, 1] payload_rows_per_dst=[16336, 16152, 9246, 12738, 16362, 16364, 16366, 16360] dup_rows_per_dst=[0, 50, 80, 12, 6, 5994, 4938, 68] combine_rows_per_src=[16336, 16152, 9246, 12738, 16362, 16364, 16366, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=874020864 dispatch_meta_bytes=458728 combine_payload_bytes=874020864 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=874479592 ingress_bytes=1578663936 link_bytes=1578663936 token_bytes=1748500456 weight_movement_bytes=704643072 total_nvlink_bytes=2453143528 weight_to_token_ratio=0.402999 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.839859075e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123858 payload_rows_remote=107478 dup_rows=7214 dup_rows_remote=6260 dedup_ratio=0.055038 router_tokens_per_expert=[8184, 0, 0, 0, 0, 0, 0, 8192, 1, 10, 5945, 8191, 0, 0, 1, 6831, 0, 0, 0, 0, 478, 8189, 0, 0, 1, 0, 3129, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16368, 16384, 11912, 13662, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 954, 6258, 0] combine_rows_per_src=[16368, 16384, 11912, 13662, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125814 payload_rows_remote=109430 dup_rows=5258 dup_rows_remote=1784 dedup_ratio=0.040115 router_tokens_per_expert=[8183, 1, 7, 0, 0, 0, 0, 8192, 0, 13, 6377, 8192, 0, 0, 0, 7376, 0, 0, 0, 0, 882, 8192, 0, 0, 0, 0, 1737, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16366, 16384, 12776, 14752, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16, 0, 4, 0, 0, 1764, 3474, 0] combine_rows_per_src=[16366, 16384, 12776, 14752, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=930938880 dispatch_meta_bytes=458520 combine_payload_bytes=930938880 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=931397400 ingress_bytes=1459421184 link_bytes=1459421184 token_bytes=1862336280 weight_movement_bytes=528482304 total_nvlink_bytes=2390818584 weight_to_token_ratio=0.283774 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.172149438e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=870432768 dispatch_meta_bytes=455752 combine_payload_bytes=870432768 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=870888520 ingress_bytes=1927397376 link_bytes=1927397376 token_bytes=1741321288 weight_movement_bytes=1056964608 total_nvlink_bytes=2798285896 weight_to_token_ratio=0.606990 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=28032 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.079262077e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=919044096 dispatch_meta_bytes=462240 combine_payload_bytes=919044096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=919506336 ingress_bytes=1271365632 link_bytes=1271365632 token_bytes=1838550432 weight_movement_bytes=352321536 total_nvlink_bytes=2190871968 weight_to_token_ratio=0.191630 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.119116845e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=715980800 dispatch_meta_bytes=458648 combine_payload_bytes=715980800 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=716439448 ingress_bytes=1244463104 link_bytes=1244463104 token_bytes=1432420248 weight_movement_bytes=528482304 total_nvlink_bytes=1960902552 weight_to_token_ratio=0.368944 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.968473918e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=848396288 dispatch_meta_bytes=458576 combine_payload_bytes=848396288 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=848854864 ingress_bytes=1905360896 link_bytes=1905360896 token_bytes=1697251152 weight_movement_bytes=1056964608 total_nvlink_bytes=2754215760 weight_to_token_ratio=0.622751 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.066922568e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=880459776 dispatch_meta_bytes=454952 combine_payload_bytes=880459776 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=880914728 ingress_bytes=1585102848 link_bytes=1585102848 token_bytes=1761374504 weight_movement_bytes=704643072 total_nvlink_bytes=2466017576 weight_to_token_ratio=0.400053 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.875914294e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=896450560 dispatch_meta_bytes=444856 combine_payload_bytes=896450560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=896895416 ingress_bytes=1248772096 link_bytes=1248772096 token_bytes=1793345976 weight_movement_bytes=352321536 total_nvlink_bytes=2145667512 weight_to_token_ratio=0.196460 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=28288 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.992602474e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130980 payload_rows_remote=114630 dup_rows=92 dup_rows_remote=92 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8175, 8192, 20, 0, 0, 8191, 0, 0, 0, 0, 0, 8192, 0, 1, 23, 0, 0, 7, 0, 8191, 0, 0, 0, 8173, 0, 8179] payload_rows_per_dst=[16384, 16350, 16382, 16384, 16384, 16368, 16382, 16346] dup_rows_per_dst=[0, 0, 40, 0, 2, 50, 0, 0] combine_rows_per_src=[16384, 16350, 16382, 16384, 16384, 16368, 16382, 16346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131014 payload_rows_remote=114632 dup_rows=58 dup_rows_remote=40 dedup_ratio=0.000443 router_tokens_per_expert=[2, 2, 8189, 1, 1, 1, 8181, 8191, 8, 1, 1, 8190, 0, 0, 1, 0, 1, 8188, 2, 7, 9, 0, 0, 1, 1, 8189, 0, 0, 3, 8185, 0, 8181] payload_rows_per_dst=[16382, 16364, 16382, 16384, 16382, 16370, 16380, 16370] dup_rows_per_dst=[6, 2, 18, 0, 14, 12, 0, 6] combine_rows_per_src=[16382, 16364, 16382, 16384, 16382, 16370, 16380, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939048960 dispatch_meta_bytes=458888 combine_payload_bytes=939048960 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939507848 ingress_bytes=1643692032 link_bytes=1643692032 token_bytes=1878556808 weight_movement_bytes=704643072 total_nvlink_bytes=2583199880 weight_to_token_ratio=0.375098 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203989268e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129020 payload_rows_remote=112654 dup_rows=2052 dup_rows_remote=2052 dedup_ratio=0.015656 router_tokens_per_expert=[8177, 2, 4, 1, 38, 0, 11, 8191, 1, 3, 7490, 8187, 0, 1, 0, 7890, 0, 0, 0, 0, 199, 8189, 0, 2, 2, 4, 772, 2, 8186, 5, 8178, 1] payload_rows_per_dst=[16364, 16384, 14988, 15780, 16366, 16384, 16382, 16372] dup_rows_per_dst=[4, 96, 0, 2, 0, 396, 1552, 2] combine_rows_per_src=[16364, 16384, 14988, 15780, 16366, 16384, 16382, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131044 payload_rows_remote=114660 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8186, 8192, 2, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 13, 0, 0, 1, 0, 8192, 0, 0, 0, 8184, 0, 8190] payload_rows_per_dst=[16384, 16372, 16384, 16384, 16384, 16384, 16384, 16368] dup_rows_per_dst=[0, 0, 4, 0, 0, 24, 0, 0] combine_rows_per_src=[16384, 16372, 16384, 16384, 16384, 16384, 16384, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130884 payload_rows_remote=114502 dup_rows=188 dup_rows_remote=170 dedup_ratio=0.001434 router_tokens_per_expert=[0, 7, 8188, 0, 2, 4, 8183, 8185, 0, 0, 3, 8187, 4, 4, 7, 1, 0, 8188, 0, 18, 12, 4, 0, 0, 0, 8180, 0, 5, 0, 8176, 0, 8178] payload_rows_per_dst=[16382, 16382, 16334, 16350, 16354, 16378, 16370, 16334] dup_rows_per_dst=[18, 106, 0, 18, 22, 10, 14, 0] combine_rows_per_src=[16382, 16382, 16334, 16350, 16354, 16378, 16370, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130774 payload_rows_remote=114410 dup_rows=298 dup_rows_remote=156 dedup_ratio=0.002274 router_tokens_per_expert=[0, 3, 8181, 3, 4, 6, 8187, 8186, 16, 0, 5, 8184, 0, 2, 6, 8, 0, 8190, 1, 11, 12, 3, 0, 1, 4, 8163, 0, 0, 2, 8180, 0, 8178] payload_rows_per_dst=[16372, 16384, 16334, 16384, 16346, 16256, 16334, 16364] dup_rows_per_dst=[0, 10, 26, 20, 24, 26, 50, 142] combine_rows_per_src=[16372, 16384, 16334, 16384, 16346, 16256, 16334, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131060 payload_rows_remote=114680 dup_rows=12 dup_rows_remote=8 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8188, 8192, 2, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 2, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190] payload_rows_per_dst=[16384, 16376, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 0, 4, 4, 0, 0] combine_rows_per_src=[16384, 16376, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8192, 8192, 2, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16382] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939065344 dispatch_meta_bytes=458688 combine_payload_bytes=939065344 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939524032 ingress_bytes=1819869184 link_bytes=1819869184 token_bytes=1878589376 weight_movement_bytes=880803840 total_nvlink_bytes=2759393216 weight_to_token_ratio=0.468864 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019050778e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=922861568 dispatch_meta_bytes=458824 combine_payload_bytes=922861568 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=923320392 ingress_bytes=1451343872 link_bytes=1451343872 token_bytes=1846181960 weight_movement_bytes=528482304 total_nvlink_bytes=2374664264 weight_to_token_ratio=0.286257 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.126919862e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=458752 combine_payload_bytes=939294720 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939753472 ingress_bytes=1291616256 link_bytes=1291616256 token_bytes=1879048192 weight_movement_bytes=352321536 total_nvlink_bytes=2231369728 weight_to_token_ratio=0.187500 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232511886e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938000384 dispatch_meta_bytes=458688 combine_payload_bytes=938000384 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938459072 ingress_bytes=1290321920 link_bytes=1290321920 token_bytes=1876459456 weight_movement_bytes=352321536 total_nvlink_bytes=2228780992 weight_to_token_ratio=0.187759 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.225264145e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=937246720 dispatch_meta_bytes=458264 combine_payload_bytes=937246720 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=937704984 ingress_bytes=1818050560 link_bytes=1818050560 token_bytes=1874951704 weight_movement_bytes=880803840 total_nvlink_bytes=2755755544 weight_to_token_ratio=0.469774 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018032424e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130544 payload_rows_remote=114166 dup_rows=528 dup_rows_remote=528 dedup_ratio=0.004028 router_tokens_per_expert=[0, 0, 7953, 8192, 0, 0, 0, 8189, 10, 0, 169, 0, 0, 0, 0, 0, 0, 65, 8191, 0, 1, 8170, 22, 8190, 8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15906, 16378, 16384, 16340, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 358, 0, 128, 42, 0, 0] combine_rows_per_src=[15906, 16378, 16384, 16340, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939917312 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879375872 weight_movement_bytes=352321536 total_nvlink_bytes=2231697408 weight_to_token_ratio=0.187467 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879441408 weight_movement_bytes=352321536 total_nvlink_bytes=2231762944 weight_to_token_ratio=0.187461 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130624 payload_rows_remote=114242 dup_rows=448 dup_rows_remote=70 dedup_ratio=0.003418 router_tokens_per_expert=[1, 2, 7991, 8188, 2, 4, 7, 8186, 28, 0, 160, 4, 2, 2, 4, 0, 0, 14, 8188, 1, 6, 8183, 1, 8177, 8190, 8186, 0, 1, 0, 4, 4, 0] payload_rows_per_dst=[15986, 16380, 16382, 16374, 16378, 16368, 16374, 16382] dup_rows_per_dst=[2, 18, 378, 8, 28, 0, 0, 14] combine_rows_per_src=[15986, 16380, 16382, 16374, 16378, 16368, 16374, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=935247872 dispatch_meta_bytes=458776 combine_payload_bytes=935247872 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=935706648 ingress_bytes=1639890944 link_bytes=1639890944 token_bytes=1870954520 weight_movement_bytes=704643072 total_nvlink_bytes=2575597592 weight_to_token_ratio=0.376622 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.182704762e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130998 payload_rows_remote=114618 dup_rows=74 dup_rows_remote=66 dedup_ratio=0.000565 router_tokens_per_expert=[0, 0, 8188, 0, 0, 1, 8170, 8187, 3, 3, 0, 8190, 0, 1, 0, 2, 0, 8189, 0, 5, 19, 0, 0, 7, 2, 8191, 1, 1, 4, 8183, 1, 8188] payload_rows_per_dst=[16376, 16342, 16382, 16376, 16380, 16384, 16384, 16374] dup_rows_per_dst=[0, 0, 10, 4, 8, 44, 6, 2] combine_rows_per_src=[16376, 16342, 16382, 16376, 16380, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130642 payload_rows_remote=114276 dup_rows=430 dup_rows_remote=430 dedup_ratio=0.003281 router_tokens_per_expert=[0, 0, 7986, 8192, 0, 0, 3, 8192, 1, 0, 65, 0, 0, 0, 0, 0, 0, 116, 8192, 0, 0, 8183, 23, 8192, 8192, 8192, 0, 0, 0, 7, 0, 0] payload_rows_per_dst=[15972, 16384, 16384, 16366, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 6, 132, 0, 232, 46, 0, 14] combine_rows_per_src=[15972, 16384, 16384, 16366, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129136 payload_rows_remote=112778 dup_rows=1936 dup_rows_remote=208 dedup_ratio=0.014771 router_tokens_per_expert=[5, 3, 8099, 8182, 2, 0, 17, 8160, 3, 0, 112, 10, 4, 7, 0, 0, 4, 30, 8187, 2, 0, 8188, 0, 8139, 8189, 8186, 0, 3, 0, 4, 0, 0] payload_rows_per_dst=[16358, 16346, 15218, 16384, 15872, 16218, 16384, 16356] dup_rows_per_dst=[1728, 60, 12, 74, 38, 0, 16, 8] combine_rows_per_src=[16358, 16346, 15218, 16384, 15872, 16218, 16384, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=935870464 dispatch_meta_bytes=457248 combine_payload_bytes=935870464 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936327712 ingress_bytes=1288192000 link_bytes=1288192000 token_bytes=1872198176 weight_movement_bytes=352321536 total_nvlink_bytes=2224519712 weight_to_token_ratio=0.188186 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.213337482e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130974 payload_rows_remote=114592 dup_rows=98 dup_rows_remote=98 dedup_ratio=0.000748 router_tokens_per_expert=[0, 0, 8148, 8192, 4, 0, 0, 8189, 0, 0, 13, 4, 0, 0, 0, 0, 0, 25, 8192, 0, 0, 8191, 0, 8192, 8192, 8191, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[16296, 16378, 16384, 16382, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 8, 34, 0, 50, 0, 0, 6] combine_rows_per_src=[16296, 16378, 16384, 16382, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130526 payload_rows_remote=114156 dup_rows=546 dup_rows_remote=546 dedup_ratio=0.004166 router_tokens_per_expert=[5, 7, 8130, 8185, 0, 3, 20, 8143, 43, 0, 5, 14, 5, 7, 0, 6, 4, 88, 8183, 4, 0, 8174, 16, 8126, 8185, 8175, 7, 1, 0, 0, 0, 0] payload_rows_per_dst=[16272, 16274, 16384, 16370, 16214, 16280, 16362, 16370] dup_rows_per_dst=[10, 24, 110, 14, 174, 192, 22, 0] combine_rows_per_src=[16272, 16274, 16384, 16370, 16214, 16280, 16362, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114648 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 8175, 8192, 12, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8192, 0, 0, 8190, 0, 8192, 8192, 8191, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[16350, 16384, 16384, 16380, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 24, 0, 0, 12, 0, 0, 4] combine_rows_per_src=[16350, 16384, 16384, 16380, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938950656 dispatch_meta_bytes=458736 combine_payload_bytes=938950656 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939409392 ingress_bytes=1995915264 link_bytes=1995915264 token_bytes=1878360048 weight_movement_bytes=1056964608 total_nvlink_bytes=2935324656 weight_to_token_ratio=0.562706 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117629234e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=936148992 dispatch_meta_bytes=458824 combine_payload_bytes=936148992 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=936607816 ingress_bytes=1640792064 link_bytes=1640792064 token_bytes=1872756808 weight_movement_bytes=704643072 total_nvlink_bytes=2577399880 weight_to_token_ratio=0.376260 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.187750658e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 4, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=923877376 dispatch_meta_bytes=451944 combine_payload_bytes=923877376 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=924329320 ingress_bytes=1628520448 link_bytes=1628520448 token_bytes=1848206696 weight_movement_bytes=704643072 total_nvlink_bytes=2552849768 weight_to_token_ratio=0.381258 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.119034731e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938737664 dispatch_meta_bytes=458760 combine_payload_bytes=938737664 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939196424 ingress_bytes=1467219968 link_bytes=1467219968 token_bytes=1877934088 weight_movement_bytes=528482304 total_nvlink_bytes=2406416392 weight_to_token_ratio=0.281417 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.215819373e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=935165952 dispatch_meta_bytes=458808 combine_payload_bytes=935165952 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935624760 ingress_bytes=1287487488 link_bytes=1287487488 token_bytes=1870790712 weight_movement_bytes=352321536 total_nvlink_bytes=2223112248 weight_to_token_ratio=0.188328 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.209392509e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130044 payload_rows_remote=113660 dup_rows=1028 dup_rows_remote=1028 dedup_ratio=0.007843 router_tokens_per_expert=[9, 8192, 2, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 10, 0, 0, 5, 0, 0, 0, 8179, 0, 7911, 7968, 0, 0, 0, 8192, 0, 492, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16366, 15822, 15936, 16384] dup_rows_per_dst=[22, 0, 0, 20, 2, 0, 0, 984] combine_rows_per_src=[16384, 16384, 16384, 16384, 16366, 15822, 15936, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458752 combine_payload_bytes=939196416 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939655168 ingress_bytes=1996161024 link_bytes=1996161024 token_bytes=1878851584 weight_movement_bytes=1056964608 total_nvlink_bytes=2935816192 weight_to_token_ratio=0.562559 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117766850e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129946 payload_rows_remote=113568 dup_rows=1126 dup_rows_remote=1122 dedup_ratio=0.008591 router_tokens_per_expert=[209, 8189, 21, 8190, 0, 1, 0, 1, 2, 6, 8183, 0, 0, 8179, 7, 1, 0, 7, 0, 0, 0, 8185, 1, 7729, 8108, 3, 0, 2, 8189, 2, 319, 2] payload_rows_per_dst=[16380, 16384, 16378, 16370, 16370, 15460, 16222, 16382] dup_rows_per_dst=[458, 0, 4, 4, 14, 0, 4, 642] combine_rows_per_src=[16380, 16384, 16378, 16370, 16370, 15460, 16222, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=931102720 dispatch_meta_bytes=458752 combine_payload_bytes=931102720 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=931561472 ingress_bytes=1459585024 link_bytes=1459585024 token_bytes=1862664192 weight_movement_bytes=528482304 total_nvlink_bytes=2391146496 weight_to_token_ratio=0.283724 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.173066874e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130254 payload_rows_remote=113870 dup_rows=818 dup_rows_remote=802 dedup_ratio=0.006241 router_tokens_per_expert=[5, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 8185, 0, 7832, 8150, 0, 0, 0, 8192, 0, 396, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16370, 15664, 16300, 16384] dup_rows_per_dst=[10, 0, 0, 16, 0, 0, 0, 792] combine_rows_per_src=[16384, 16384, 16384, 16384, 16370, 15664, 16300, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124440 payload_rows_remote=108062 dup_rows=6632 dup_rows_remote=6154 dedup_ratio=0.050598 router_tokens_per_expert=[159, 8183, 48, 8188, 0, 0, 0, 4, 5, 7, 8178, 0, 0, 8149, 3, 0, 0, 0, 6, 3, 0, 8148, 4, 7895, 8108, 0, 3, 1, 8186, 6, 248, 4] payload_rows_per_dst=[16378, 16370, 16372, 16298, 16384, 16220, 16220, 10198] dup_rows_per_dst=[478, 0, 8, 6, 146, 5888, 4, 102] combine_rows_per_src=[16378, 16370, 16372, 16298, 16384, 16220, 16220, 10198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130858 payload_rows_remote=114474 dup_rows=214 dup_rows_remote=188 dedup_ratio=0.001633 router_tokens_per_expert=[1, 1, 8094, 8191, 18, 0, 0, 8186, 26, 4, 40, 3, 1, 0, 0, 0, 1, 12, 8191, 1, 0, 8189, 5, 8190, 8190, 8190, 0, 1, 0, 0, 1, 0] payload_rows_per_dst=[16190, 16376, 16382, 16380, 16384, 16382, 16382, 16382] dup_rows_per_dst=[2, 32, 146, 0, 26, 8, 0, 0] combine_rows_per_src=[16190, 16376, 16382, 16380, 16384, 16382, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=930349056 dispatch_meta_bytes=458760 combine_payload_bytes=930349056 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=930807816 ingress_bytes=1282670592 link_bytes=1282670592 token_bytes=1861156872 weight_movement_bytes=352321536 total_nvlink_bytes=2213478408 weight_to_token_ratio=0.189302 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.182419902e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 3, 0, 2, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130856 payload_rows_remote=114546 dup_rows=216 dup_rows_remote=216 dedup_ratio=0.001648 router_tokens_per_expert=[37, 8192, 2, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8126, 8155, 0, 0, 0, 8192, 0, 69, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16378, 16252, 16310, 16384] dup_rows_per_dst=[78, 0, 0, 0, 0, 0, 0, 138] combine_rows_per_src=[16384, 16384, 16384, 16380, 16378, 16252, 16310, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125490 payload_rows_remote=109112 dup_rows=5582 dup_rows_remote=3164 dedup_ratio=0.042587 router_tokens_per_expert=[605, 8180, 236, 8182, 0, 8, 1, 4, 6, 8, 8179, 2, 0, 8119, 56, 0, 0, 0, 7, 4, 0, 8154, 7, 6992, 8101, 1, 0, 7, 8184, 5, 484, 4] payload_rows_per_dst=[13544, 16376, 16384, 16274, 16320, 13998, 16216, 16378] dup_rows_per_dst=[1308, 22, 6, 420, 10, 0, 1398, 2418] combine_rows_per_src=[13544, 16376, 16384, 16274, 16320, 13998, 16216, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130296 payload_rows_remote=114512 dup_rows=776 dup_rows_remote=776 dedup_ratio=0.005920 router_tokens_per_expert=[68, 8192, 8, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8115, 2, 0, 0, 0, 0, 0, 0, 8183, 0, 7892, 8189, 0, 0, 0, 8192, 0, 311, 0] payload_rows_per_dst=[16384, 16384, 16384, 16232, 16366, 15784, 16378, 16384] dup_rows_per_dst=[152, 0, 0, 2, 0, 0, 0, 622] combine_rows_per_src=[16384, 16384, 16384, 16232, 16366, 15784, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=932823040 dispatch_meta_bytes=458688 combine_payload_bytes=932823040 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933281728 ingress_bytes=1285144576 link_bytes=1285144576 token_bytes=1866104768 weight_movement_bytes=352321536 total_nvlink_bytes=2218426304 weight_to_token_ratio=0.188801 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.196273180e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=885243904 dispatch_meta_bytes=456864 combine_payload_bytes=885243904 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=885700768 ingress_bytes=2646851584 link_bytes=2646851584 token_bytes=1770944672 weight_movement_bytes=1761607680 total_nvlink_bytes=3532552352 weight_to_token_ratio=0.994728 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.482126402e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=937771008 dispatch_meta_bytes=458648 combine_payload_bytes=937771008 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=938229656 ingress_bytes=1818574848 link_bytes=1818574848 token_bytes=1876000664 weight_movement_bytes=880803840 total_nvlink_bytes=2756804504 weight_to_token_ratio=0.469511 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018326004e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=893845504 dispatch_meta_bytes=449104 combine_payload_bytes=893845504 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=894294608 ingress_bytes=1774649344 link_bytes=1774649344 token_bytes=1788140112 weight_movement_bytes=880803840 total_nvlink_bytes=2668943952 weight_to_token_ratio=0.492581 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.937295551e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938360832 dispatch_meta_bytes=459048 combine_payload_bytes=938360832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938819880 ingress_bytes=1290682368 link_bytes=1290682368 token_bytes=1877180712 weight_movement_bytes=352321536 total_nvlink_bytes=2229502248 weight_to_token_ratio=0.187687 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227282503e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130836 payload_rows_remote=114452 dup_rows=236 dup_rows_remote=236 dedup_ratio=0.001801 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 8187, 8192, 8192, 0, 0, 0, 0, 0, 22, 0, 8174, 118, 8192, 0, 8074, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16148, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 34, 200, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16148, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938082304 dispatch_meta_bytes=461152 combine_payload_bytes=938082304 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=938543456 ingress_bytes=1995046912 link_bytes=1995046912 token_bytes=1876625760 weight_movement_bytes=1056964608 total_nvlink_bytes=2933590368 weight_to_token_ratio=0.563226 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117142993e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130942 payload_rows_remote=114564 dup_rows=130 dup_rows_remote=130 dedup_ratio=0.000992 router_tokens_per_expert=[8186, 2, 0, 0, 8183, 0, 1, 8187, 8188, 8189, 0, 0, 0, 0, 6, 18, 0, 8188, 26, 8188, 2, 8144, 3, 5, 2, 4, 3, 8, 0, 3, 0, 0] payload_rows_per_dst=[16376, 16368, 16378, 16380, 16378, 16296, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 42, 50, 12, 26, 0] combine_rows_per_src=[16376, 16368, 16378, 16380, 16378, 16296, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=937590784 dispatch_meta_bytes=458752 combine_payload_bytes=937590784 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=938049536 ingress_bytes=1994555392 link_bytes=1994555392 token_bytes=1875640320 weight_movement_bytes=1056964608 total_nvlink_bytes=2932604928 weight_to_token_ratio=0.563522 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.116867763e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130696 payload_rows_remote=114312 dup_rows=376 dup_rows_remote=340 dedup_ratio=0.002869 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 18, 0, 8187, 175, 8192, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16008, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 36, 340, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16008, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130300 payload_rows_remote=113918 dup_rows=772 dup_rows_remote=744 dedup_ratio=0.005890 router_tokens_per_expert=[8185, 4, 0, 5, 8182, 3, 3, 8185, 8185, 8182, 0, 2, 0, 0, 0, 22, 1, 8185, 126, 8181, 6, 8008, 6, 0, 6, 2, 7, 38, 0, 6, 0, 6] payload_rows_per_dst=[16382, 16380, 16374, 16120, 16372, 16028, 16270, 16374] dup_rows_per_dst=[28, 412, 40, 0, 256, 12, 16, 8] combine_rows_per_src=[16382, 16380, 16374, 16120, 16372, 16028, 16270, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938508288 dispatch_meta_bytes=458776 combine_payload_bytes=938508288 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=938967064 ingress_bytes=1819312128 link_bytes=1819312128 token_bytes=1877475352 weight_movement_bytes=880803840 total_nvlink_bytes=2758279192 weight_to_token_ratio=0.469143 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018738850e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130224 payload_rows_remote=113880 dup_rows=848 dup_rows_remote=846 dedup_ratio=0.006470 router_tokens_per_expert=[174, 8190, 0, 8191, 0, 0, 0, 1, 1, 0, 8191, 0, 0, 8140, 1, 0, 0, 4, 2, 1, 0, 8166, 0, 7992, 8039, 1, 2, 0, 8191, 1, 247, 1] payload_rows_per_dst=[16382, 16384, 16384, 16280, 16344, 15984, 16082, 16384] dup_rows_per_dst=[346, 0, 0, 2, 2, 0, 2, 496] combine_rows_per_src=[16382, 16384, 16384, 16280, 16344, 15984, 16082, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128150 payload_rows_remote=111792 dup_rows=2922 dup_rows_remote=2896 dedup_ratio=0.022293 router_tokens_per_expert=[8180, 10, 0, 8, 8162, 2, 4, 8185, 8184, 8182, 0, 0, 4, 0, 27, 18, 1, 8165, 224, 8175, 7, 7967, 7, 6, 7, 0, 5, 2, 1, 3, 0, 0] payload_rows_per_dst=[16348, 15988, 16364, 16372, 14398, 15938, 16384, 16358] dup_rows_per_dst=[22, 2, 0, 96, 414, 2350, 12, 26] combine_rows_per_src=[16348, 15988, 16364, 16372, 14398, 15938, 16384, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130956 payload_rows_remote=114572 dup_rows=116 dup_rows_remote=114 dedup_ratio=0.000885 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 18, 0, 8182, 49, 8192, 0, 8134, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16268, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 36, 78, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16268, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=936443904 dispatch_meta_bytes=458608 combine_payload_bytes=936443904 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936902512 ingress_bytes=1288765440 link_bytes=1288765440 token_bytes=1873346416 weight_movement_bytes=352321536 total_nvlink_bytes=2225667952 weight_to_token_ratio=0.188071 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.216548507e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130728 payload_rows_remote=114688 dup_rows=344 dup_rows_remote=344 dedup_ratio=0.002625 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8163, 201, 8192, 0, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16040, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 344, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16040, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=933216256 dispatch_meta_bytes=458648 combine_payload_bytes=933216256 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=933674904 ingress_bytes=1461698560 link_bytes=1461698560 token_bytes=1866891160 weight_movement_bytes=528482304 total_nvlink_bytes=2395373464 weight_to_token_ratio=0.283081 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.184901793e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=932904960 dispatch_meta_bytes=458904 combine_payload_bytes=932904960 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=933363864 ingress_bytes=1637548032 link_bytes=1637548032 token_bytes=1866268824 weight_movement_bytes=704643072 total_nvlink_bytes=2570911896 weight_to_token_ratio=0.377568 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.169585433e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=915800064 dispatch_meta_bytes=458752 combine_payload_bytes=915800064 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=916258816 ingress_bytes=1444282368 link_bytes=1444282368 token_bytes=1832058880 weight_movement_bytes=528482304 total_nvlink_bytes=2360541184 weight_to_token_ratio=0.288464 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.087378388e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131006 payload_rows_remote=114622 dup_rows=66 dup_rows_remote=56 dedup_ratio=0.000504 router_tokens_per_expert=[0, 0, 0, 0, 8192, 5, 0, 0, 0, 14, 0, 8187, 0, 0, 0, 8192, 8192, 0, 0, 0, 16, 8192, 8192, 0, 0, 8162, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16384, 16384, 16324, 16384] dup_rows_per_dst=[0, 10, 24, 0, 0, 32, 0, 0] combine_rows_per_src=[16384, 16384, 16378, 16384, 16384, 16384, 16324, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938573824 dispatch_meta_bytes=458744 combine_payload_bytes=938573824 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939032568 ingress_bytes=1290895360 link_bytes=1290895360 token_bytes=1877606392 weight_movement_bytes=352321536 total_nvlink_bytes=2229927928 weight_to_token_ratio=0.187644 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228475170e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=460128 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939984224 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879508320 weight_movement_bytes=352321536 total_nvlink_bytes=2231829856 weight_to_token_ratio=0.187454 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130994 payload_rows_remote=114630 dup_rows=78 dup_rows_remote=78 dedup_ratio=0.000595 router_tokens_per_expert=[2, 2, 5, 0, 8190, 29, 2, 3, 0, 12, 3, 8167, 0, 0, 0, 8187, 8189, 0, 3, 0, 0, 8189, 8189, 0, 0, 8169, 8190, 3, 0, 0, 0, 2] payload_rows_per_dst=[16384, 16384, 16364, 16374, 16384, 16378, 16344, 16382] dup_rows_per_dst=[14, 64, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16364, 16374, 16384, 16378, 16344, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=458712 combine_payload_bytes=938983424 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939442136 ingress_bytes=1819787264 link_bytes=1819787264 token_bytes=1878425560 weight_movement_bytes=880803840 total_nvlink_bytes=2759229400 weight_to_token_ratio=0.468905 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019004906e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114648 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 8192, 18, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 1, 8192, 8192, 0, 0, 8172, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16344, 16384] dup_rows_per_dst=[0, 36, 2, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16344, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939048960 dispatch_meta_bytes=458832 combine_payload_bytes=939048960 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=939507792 ingress_bytes=1643692032 link_bytes=1643692032 token_bytes=1878556752 weight_movement_bytes=704643072 total_nvlink_bytes=2583199824 weight_to_token_ratio=0.375098 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203989268e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130888 payload_rows_remote=114504 dup_rows=184 dup_rows_remote=164 dedup_ratio=0.001404 router_tokens_per_expert=[0, 7, 10, 1, 8186, 13, 7, 6, 0, 41, 8, 8160, 0, 0, 8, 8165, 8183, 0, 5, 0, 0, 8185, 8185, 2, 0, 8151, 8184, 8, 0, 16, 0, 5] payload_rows_per_dst=[16384, 16384, 16384, 16344, 16376, 16342, 16318, 16356] dup_rows_per_dst=[20, 118, 34, 2, 0, 4, 0, 6] combine_rows_per_src=[16384, 16384, 16384, 16344, 16376, 16342, 16318, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130602 payload_rows_remote=114266 dup_rows=470 dup_rows_remote=100 dedup_ratio=0.003586 router_tokens_per_expert=[8190, 0, 0, 0, 8191, 1, 1, 8184, 8191, 8190, 0, 0, 1, 0, 0, 25, 1, 8122, 230, 8191, 1, 7985, 0, 0, 0, 0, 1, 28, 0, 1, 0, 2] payload_rows_per_dst=[16380, 16384, 16380, 16382, 16336, 15972, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 38, 370, 0, 56, 4] combine_rows_per_src=[16380, 16384, 16380, 16382, 16336, 15972, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130510 payload_rows_remote=114132 dup_rows=562 dup_rows_remote=540 dedup_ratio=0.004288 router_tokens_per_expert=[0, 8, 7, 3, 8185, 38, 4, 7, 0, 20, 9, 8169, 0, 0, 7, 8177, 8183, 0, 6, 1, 1, 8183, 8184, 8, 1, 8127, 8182, 10, 0, 8, 3, 5] payload_rows_per_dst=[16326, 16042, 16366, 16364, 16378, 16382, 16274, 16378] dup_rows_per_dst=[24, 84, 12, 54, 8, 14, 344, 22] combine_rows_per_src=[16326, 16042, 16366, 16364, 16378, 16382, 16274, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114688 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 8191, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458752 combine_payload_bytes=939196416 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939655168 ingress_bytes=1115357184 link_bytes=1115357184 token_bytes=1878851584 weight_movement_bytes=176160768 total_nvlink_bytes=2055012352 weight_to_token_ratio=0.093760 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245534657e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131050 payload_rows_remote=114666 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000168 router_tokens_per_expert=[0, 0, 0, 0, 8192, 2, 0, 0, 0, 1, 0, 8192, 0, 0, 8, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 8181, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16362, 16384] dup_rows_per_dst=[0, 4, 2, 16, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16362, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 3] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938016768 dispatch_meta_bytes=458672 combine_payload_bytes=938016768 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938475440 ingress_bytes=1290338304 link_bytes=1290338304 token_bytes=1876492208 weight_movement_bytes=352321536 total_nvlink_bytes=2228813744 weight_to_token_ratio=0.187755 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.225355889e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130654 payload_rows_remote=114282 dup_rows=418 dup_rows_remote=418 dedup_ratio=0.003189 router_tokens_per_expert=[0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 5, 8188, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 7996, 8186, 0, 0, 8192, 8166, 0, 230] payload_rows_per_dst=[16382, 16372, 16376, 16384, 16380, 16384, 15992, 16384] dup_rows_per_dst=[0, 0, 10, 0, 0, 0, 0, 408] combine_rows_per_src=[16382, 16372, 16376, 16384, 16380, 16384, 15992, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=936067072 dispatch_meta_bytes=457464 combine_payload_bytes=936067072 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=936524536 ingress_bytes=2169192448 link_bytes=2169192448 token_bytes=1872591608 weight_movement_bytes=1233125376 total_nvlink_bytes=3105716984 weight_to_token_ratio=0.658513 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.214657224e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=934969344 dispatch_meta_bytes=458688 combine_payload_bytes=934969344 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935428032 ingress_bytes=1287290880 link_bytes=1287290880 token_bytes=1870397376 weight_movement_bytes=352321536 total_nvlink_bytes=2222718912 weight_to_token_ratio=0.188367 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.208291586e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458760 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939982856 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879506952 weight_movement_bytes=704643072 total_nvlink_bytes=2584150024 weight_to_token_ratio=0.374908 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130922 payload_rows_remote=114546 dup_rows=150 dup_rows_remote=142 dedup_ratio=0.001144 router_tokens_per_expert=[0, 1, 8188, 8185, 0, 0, 2, 3, 0, 0, 4, 8188, 0, 3, 1, 0, 0, 8189, 0, 5, 5, 0, 5, 2, 8129, 8190, 0, 3, 8189, 8173, 3, 68] payload_rows_per_dst=[16372, 16384, 16376, 16384, 16384, 16384, 16264, 16374] dup_rows_per_dst=[0, 6, 8, 2, 4, 16, 0, 114] combine_rows_per_src=[16372, 16384, 16376, 16384, 16384, 16384, 16264, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=458752 combine_payload_bytes=939343872 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939802624 ingress_bytes=1996308480 link_bytes=1996308480 token_bytes=1879146496 weight_movement_bytes=1056964608 total_nvlink_bytes=2936111104 weight_to_token_ratio=0.562471 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117849419e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936198144 dispatch_meta_bytes=458800 combine_payload_bytes=936198144 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936656944 ingress_bytes=1288519680 link_bytes=1288519680 token_bytes=1872855088 weight_movement_bytes=352321536 total_nvlink_bytes=2225176624 weight_to_token_ratio=0.188120 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.215172353e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938360832 dispatch_meta_bytes=458752 combine_payload_bytes=938360832 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938819584 ingress_bytes=1466843136 link_bytes=1466843136 token_bytes=1877180416 weight_movement_bytes=528482304 total_nvlink_bytes=2405662720 weight_to_token_ratio=0.281530 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.213709271e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130898 payload_rows_remote=114514 dup_rows=174 dup_rows_remote=174 dedup_ratio=0.001328 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8105, 8192, 0, 0, 8192, 8174, 0, 105] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16210, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 174] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16210, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129244 payload_rows_remote=112872 dup_rows=1828 dup_rows_remote=1696 dedup_ratio=0.013947 router_tokens_per_expert=[0, 1, 8180, 8185, 0, 5, 6, 7, 3, 49, 0, 8185, 8, 7, 0, 0, 0, 8184, 0, 12, 3, 0, 0, 5, 8158, 8187, 0, 4, 8185, 8130, 0, 32] payload_rows_per_dst=[16372, 16384, 16320, 16384, 16372, 16368, 16362, 14682] dup_rows_per_dst=[132, 26, 98, 16, 10, 0, 1542, 4] combine_rows_per_src=[16372, 16384, 16320, 16384, 16372, 16368, 16362, 14682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130892 payload_rows_remote=114646 dup_rows=180 dup_rows_remote=180 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 8192, 8171, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8123, 8192, 0, 0, 8192, 8192, 0, 90] payload_rows_per_dst=[16342, 16384, 16384, 16384, 16384, 16384, 16246, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 180] combine_rows_per_src=[16342, 16384, 16384, 16384, 16384, 16384, 16246, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130560 payload_rows_remote=114256 dup_rows=512 dup_rows_remote=162 dedup_ratio=0.003906 router_tokens_per_expert=[2, 4, 8176, 8184, 0, 4, 7, 8, 3, 29, 1, 8141, 7, 8, 0, 0, 0, 8185, 0, 16, 6, 0, 2, 5, 8070, 8182, 0, 6, 8184, 8144, 0, 162] payload_rows_per_dst=[16374, 16340, 16286, 16356, 16374, 16374, 16152, 16304] dup_rows_per_dst=[4, 26, 56, 10, 20, 10, 36, 350] combine_rows_per_src=[16374, 16340, 16286, 16356, 16374, 16374, 16152, 16304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114648 dup_rows=40 dup_rows_remote=34 dedup_ratio=0.000305 router_tokens_per_expert=[0, 1, 0, 1, 8188, 14, 1, 1, 0, 1, 1, 8175, 0, 0, 0, 8191, 8188, 6, 1, 0, 0, 8191, 8191, 0, 0, 8189, 8190, 1, 0, 1, 3, 1] payload_rows_per_dst=[16382, 16384, 16354, 16382, 16384, 16382, 16380, 16384] dup_rows_per_dst=[2, 24, 0, 0, 6, 0, 0, 8] combine_rows_per_src=[16382, 16384, 16354, 16382, 16384, 16382, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938098688 dispatch_meta_bytes=458752 combine_payload_bytes=938098688 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=938557440 ingress_bytes=1466580992 link_bytes=1466580992 token_bytes=1876656128 weight_movement_bytes=528482304 total_nvlink_bytes=2405138432 weight_to_token_ratio=0.281608 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.212241374e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130758 payload_rows_remote=114374 dup_rows=314 dup_rows_remote=314 dedup_ratio=0.002396 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8035, 8192, 0, 0, 8192, 8192, 0, 157] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16070, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 314] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16070, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130986 payload_rows_remote=114638 dup_rows=86 dup_rows_remote=86 dedup_ratio=0.000656 router_tokens_per_expert=[0, 8192, 35, 0, 8192, 0, 8192, 8174, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8146, 0, 22, 0, 0, 0, 0, 0, 0, 8191, 8, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16348, 16384, 16384, 16336, 16384, 16382, 16384] dup_rows_per_dst=[70, 0, 0, 0, 0, 0, 16, 0] combine_rows_per_src=[16384, 16348, 16384, 16384, 16336, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=924647424 dispatch_meta_bytes=458272 combine_payload_bytes=924647424 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=925105696 ingress_bytes=1805451264 link_bytes=1805451264 token_bytes=1849753120 weight_movement_bytes=880803840 total_nvlink_bytes=2730556960 weight_to_token_ratio=0.476174 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.010977345e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939180032 dispatch_meta_bytes=459304 combine_payload_bytes=939180032 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939639336 ingress_bytes=1643823104 link_bytes=1643823104 token_bytes=1878819368 weight_movement_bytes=704643072 total_nvlink_bytes=2583462440 weight_to_token_ratio=0.375046 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204723217e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130968 payload_rows_remote=114584 dup_rows=104 dup_rows_remote=92 dedup_ratio=0.000793 router_tokens_per_expert=[1, 8188, 35, 3, 8187, 0, 8189, 8176, 3, 5, 0, 3, 8188, 3, 2, 2, 2, 8159, 2, 1, 6, 0, 0, 1, 0, 0, 8185, 6, 8180, 0, 9, 0] payload_rows_per_dst=[16384, 16352, 16384, 16382, 16328, 16384, 16376, 16378] dup_rows_per_dst=[70, 0, 12, 8, 0, 8, 6, 0] combine_rows_per_src=[16384, 16352, 16384, 16382, 16328, 16384, 16376, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=935985152 dispatch_meta_bytes=457672 combine_payload_bytes=935985152 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=936442824 ingress_bytes=2169110528 link_bytes=2169110528 token_bytes=1872427976 weight_movement_bytes=1233125376 total_nvlink_bytes=3105553352 weight_to_token_ratio=0.658570 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.214611353e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458728 combine_payload_bytes=939196416 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939655144 ingress_bytes=1291517952 link_bytes=1291517952 token_bytes=1878851560 weight_movement_bytes=352321536 total_nvlink_bytes=2231173096 weight_to_token_ratio=0.187520 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231961425e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936951808 dispatch_meta_bytes=458752 combine_payload_bytes=936951808 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937410560 ingress_bytes=1289273344 link_bytes=1289273344 token_bytes=1874362368 weight_movement_bytes=352321536 total_nvlink_bytes=2226683904 weight_to_token_ratio=0.187969 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.219392557e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=458896 combine_payload_bytes=939114496 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939573392 ingress_bytes=1996079104 link_bytes=1996079104 token_bytes=1878687888 weight_movement_bytes=1056964608 total_nvlink_bytes=2935652496 weight_to_token_ratio=0.562608 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117720978e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938672128 dispatch_meta_bytes=458704 combine_payload_bytes=938672128 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939130832 ingress_bytes=1290993664 link_bytes=1290993664 token_bytes=1877802960 weight_movement_bytes=352321536 total_nvlink_bytes=2230124496 weight_to_token_ratio=0.187624 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229025631e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 8192, 1, 0, 8192, 0, 8192, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8171, 0, 22, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130860 payload_rows_remote=114476 dup_rows=212 dup_rows_remote=170 dedup_ratio=0.001617 router_tokens_per_expert=[4, 8183, 17, 9, 8184, 0, 8184, 8158, 0, 13, 0, 6, 8182, 4, 7, 0, 8, 8177, 7, 5, 1, 0, 0, 7, 0, 0, 8181, 3, 8181, 0, 15, 0] payload_rows_per_dst=[16384, 16352, 16326, 16378, 16382, 16352, 16366, 16320] dup_rows_per_dst=[42, 122, 12, 8, 18, 0, 2, 8] combine_rows_per_src=[16384, 16352, 16326, 16378, 16382, 16352, 16366, 16320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131054 payload_rows_remote=114670 dup_rows=18 dup_rows_remote=0 dedup_ratio=0.000137 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 9, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16366, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 18, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16366, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939950080 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879441408 weight_movement_bytes=528482304 total_nvlink_bytes=2407923712 weight_to_token_ratio=0.281191 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130660 payload_rows_remote=114278 dup_rows=412 dup_rows_remote=352 dedup_ratio=0.003143 router_tokens_per_expert=[0, 8180, 2, 10, 8182, 0, 8184, 8160, 0, 17, 0, 9, 8182, 5, 8, 0, 4, 8162, 7, 11, 9, 0, 4, 6, 1, 1, 8175, 0, 8183, 0, 34, 0] payload_rows_per_dst=[16108, 16320, 16384, 16366, 16362, 16384, 16354, 16382] dup_rows_per_dst=[4, 126, 32, 10, 138, 22, 20, 60] combine_rows_per_src=[16108, 16320, 16384, 16366, 16362, 16384, 16354, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130804 payload_rows_remote=114428 dup_rows=268 dup_rows_remote=268 dedup_ratio=0.002045 router_tokens_per_expert=[0, 1, 8190, 8162, 2, 0, 1, 1, 0, 0, 6, 8189, 0, 1, 0, 0, 0, 8187, 0, 1, 1, 0, 0, 1, 8098, 8187, 0, 1, 8191, 8180, 2, 134] payload_rows_per_dst=[16326, 16376, 16380, 16384, 16376, 16384, 16198, 16380] dup_rows_per_dst=[0, 6, 10, 0, 0, 0, 0, 252] combine_rows_per_src=[16326, 16376, 16380, 16384, 16376, 16384, 16198, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127092 payload_rows_remote=112020 dup_rows=3980 dup_rows_remote=1618 dedup_ratio=0.030365 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 7530, 0, 8192, 0, 8015, 0, 39, 0, 0, 8192, 682, 0, 0, 1, 8172, 0, 0, 8180, 8192, 0, 0, 4, 8192, 0, 145, 0] payload_rows_per_dst=[16360, 15072, 15240, 16384, 14924, 16344, 16384, 16384] dup_rows_per_dst=[0, 2362, 0, 78, 1242, 0, 8, 290] combine_rows_per_src=[16360, 15072, 15240, 16384, 14924, 16344, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937787392 dispatch_meta_bytes=458584 combine_payload_bytes=937787392 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=938245976 ingress_bytes=1642430464 link_bytes=1642430464 token_bytes=1876033368 weight_movement_bytes=704643072 total_nvlink_bytes=2580676440 weight_to_token_ratio=0.375603 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196925014e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130564 payload_rows_remote=114304 dup_rows=508 dup_rows_remote=508 dedup_ratio=0.003876 router_tokens_per_expert=[4, 0, 0, 0, 2, 0, 8023, 0, 8188, 1, 8126, 3, 21, 37, 4, 8188, 139, 4, 1, 7, 8175, 0, 2, 8183, 8189, 5, 5, 0, 8183, 0, 42, 4] payload_rows_per_dst=[16374, 16050, 16260, 16384, 16384, 16354, 16384, 16374] dup_rows_per_dst=[0, 0, 0, 116, 294, 0, 14, 84] combine_rows_per_src=[16374, 16050, 16260, 16384, 16384, 16354, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=458680 combine_payload_bytes=939376640 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939835320 ingress_bytes=1291698176 link_bytes=1291698176 token_bytes=1879211960 weight_movement_bytes=352321536 total_nvlink_bytes=2231533496 weight_to_token_ratio=0.187484 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232970604e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=936165376 dispatch_meta_bytes=458520 combine_payload_bytes=936165376 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=936623896 ingress_bytes=1464647680 link_bytes=1464647680 token_bytes=1872789272 weight_movement_bytes=528482304 total_nvlink_bytes=2401271576 weight_to_token_ratio=0.282190 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.201415634e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937394176 dispatch_meta_bytes=458784 combine_payload_bytes=937394176 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=937852960 ingress_bytes=1642037248 link_bytes=1642037248 token_bytes=1875247136 weight_movement_bytes=704643072 total_nvlink_bytes=2579890208 weight_to_token_ratio=0.375760 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.194723169e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966464 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474176 weight_movement_bytes=352321536 total_nvlink_bytes=2231795712 weight_to_token_ratio=0.187458 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=917667840 dispatch_meta_bytes=454552 combine_payload_bytes=917667840 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=918122392 ingress_bytes=1269989376 link_bytes=1269989376 token_bytes=1835790232 weight_movement_bytes=352321536 total_nvlink_bytes=2188111768 weight_to_token_ratio=0.191918 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=29056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.111410386e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936378368 dispatch_meta_bytes=459248 combine_payload_bytes=936378368 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=936837616 ingress_bytes=1993342976 link_bytes=1993342976 token_bytes=1873215984 weight_movement_bytes=1056964608 total_nvlink_bytes=2930180592 weight_to_token_ratio=0.564251 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=28416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.116188860e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130502 payload_rows_remote=114118 dup_rows=570 dup_rows_remote=554 dedup_ratio=0.004349 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8030, 0, 8192, 0, 8077, 0, 8, 0, 0, 8192, 185, 0, 0, 0, 8187, 0, 0, 8189, 8192, 0, 0, 0, 8192, 0, 92, 0] payload_rows_per_dst=[16378, 16060, 16154, 16384, 16384, 16374, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 16, 370, 0, 0, 184] combine_rows_per_src=[16378, 16060, 16154, 16384, 16384, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113822 payload_rows_remote=97438 dup_rows=17250 dup_rows_remote=17202 dedup_ratio=0.131607 router_tokens_per_expert=[9, 0, 0, 0, 5, 0, 6403, 0, 8184, 0, 8098, 9, 17, 0, 8, 8184, 1257, 10, 2, 2, 8176, 0, 6, 8183, 8186, 10, 2, 0, 8181, 0, 591, 13] payload_rows_per_dst=[16384, 16192, 15024, 16384, 4052, 16316, 16332, 13138] dup_rows_per_dst=[48, 12264, 4292, 98, 406, 0, 0, 142] combine_rows_per_src=[16384, 16192, 15024, 16384, 4052, 16316, 16332, 13138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129070 payload_rows_remote=112686 dup_rows=2002 dup_rows_remote=2002 dedup_ratio=0.015274 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 7297, 0, 8192, 0, 8092, 0, 0, 6, 0, 8192, 968, 0, 0, 0, 8186, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 27, 0] payload_rows_per_dst=[16384, 14594, 16184, 16384, 16384, 16372, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 12, 1936, 0, 0, 54] combine_rows_per_src=[16384, 14594, 16184, 16384, 16384, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=934854656 dispatch_meta_bytes=458688 combine_payload_bytes=934854656 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=935313344 ingress_bytes=1991819264 link_bytes=1991819264 token_bytes=1870168000 weight_movement_bytes=1056964608 total_nvlink_bytes=2927132608 weight_to_token_ratio=0.565171 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.115335645e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127054 payload_rows_remote=110670 dup_rows=4018 dup_rows_remote=1848 dedup_ratio=0.030655 router_tokens_per_expert=[10, 0, 0, 0, 7, 0, 6873, 0, 8182, 0, 7785, 10, 6, 76, 9, 8182, 712, 11, 2, 8, 8164, 0, 13, 8177, 8185, 9, 0, 5, 8180, 0, 915, 15] payload_rows_per_dst=[16374, 13760, 15590, 15828, 16384, 16354, 16380, 16384] dup_rows_per_dst=[0, 0, 0, 160, 1446, 224, 18, 2170] combine_rows_per_src=[16374, 13760, 15590, 15828, 16384, 16354, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130976 payload_rows_remote=114602 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.000732 router_tokens_per_expert=[2, 8190, 43, 1, 8190, 0, 8191, 8152, 0, 2, 0, 1, 8191, 1, 0, 0, 0, 8179, 1, 7, 0, 0, 1, 1, 0, 0, 8189, 0, 8191, 0, 3, 0] payload_rows_per_dst=[16384, 16304, 16384, 16384, 16374, 16384, 16378, 16384] dup_rows_per_dst=[88, 0, 2, 0, 0, 2, 0, 4] combine_rows_per_src=[16384, 16304, 16384, 16384, 16374, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126534 payload_rows_remote=110154 dup_rows=4538 dup_rows_remote=4538 dedup_ratio=0.034622 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 5980, 0, 8192, 0, 8137, 0, 3, 3, 0, 8192, 2152, 0, 0, 0, 8190, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 111, 0] payload_rows_per_dst=[16384, 11960, 16274, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 12, 4304, 0, 0, 222] combine_rows_per_src=[16384, 11960, 16274, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=798212096 dispatch_meta_bytes=458560 combine_payload_bytes=798212096 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=798670656 ingress_bytes=1502855168 link_bytes=1502855168 token_bytes=1596882752 weight_movement_bytes=704643072 total_nvlink_bytes=2301525824 weight_to_token_ratio=0.441262 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.415361618e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130952 payload_rows_remote=114568 dup_rows=120 dup_rows_remote=120 dedup_ratio=0.000916 router_tokens_per_expert=[0, 8192, 8132, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16264, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 120, 0, 0, 0, 0] combine_rows_per_src=[16264, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130878 payload_rows_remote=114494 dup_rows=194 dup_rows_remote=168 dedup_ratio=0.001480 router_tokens_per_expert=[0, 8186, 8113, 3, 0, 0, 6, 8176, 3, 1, 13, 8188, 0, 2, 0, 75, 1, 1, 0, 5, 0, 3, 8188, 4, 8183, 3, 0, 4, 8187, 0, 8185, 6] payload_rows_per_dst=[16232, 16364, 16384, 16378, 16384, 16384, 16374, 16378] dup_rows_per_dst=[0, 0, 26, 148, 4, 6, 6, 4] combine_rows_per_src=[16232, 16364, 16384, 16378, 16384, 16384, 16374, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=923123712 dispatch_meta_bytes=458752 combine_payload_bytes=923123712 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=923582464 ingress_bytes=1275445248 link_bytes=1275445248 token_bytes=1846706176 weight_movement_bytes=352321536 total_nvlink_bytes=2199027712 weight_to_token_ratio=0.190784 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.141960992e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=906608640 dispatch_meta_bytes=450072 combine_payload_bytes=906608640 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=907058712 ingress_bytes=1611251712 link_bytes=1611251712 token_bytes=1813667352 weight_movement_bytes=704643072 total_nvlink_bytes=2518310424 weight_to_token_ratio=0.388518 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.022337018e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938819584 dispatch_meta_bytes=458792 combine_payload_bytes=938819584 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939278376 ingress_bytes=1291141120 link_bytes=1291141120 token_bytes=1878097960 weight_movement_bytes=352321536 total_nvlink_bytes=2230419496 weight_to_token_ratio=0.187595 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229851323e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=902381568 dispatch_meta_bytes=458768 combine_payload_bytes=902381568 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=902840336 ingress_bytes=1959346176 link_bytes=1959346176 token_bytes=1805221904 weight_movement_bytes=1056964608 total_nvlink_bytes=2862186512 weight_to_token_ratio=0.585504 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.097152071e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130974 payload_rows_remote=114590 dup_rows=98 dup_rows_remote=2 dedup_ratio=0.000748 router_tokens_per_expert=[0, 8192, 8143, 0, 0, 0, 0, 8192, 0, 0, 1, 8192, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16286, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 96, 0, 0, 0, 0] combine_rows_per_src=[16286, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938541056 dispatch_meta_bytes=458752 combine_payload_bytes=938541056 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938999808 ingress_bytes=1290862592 link_bytes=1290862592 token_bytes=1877540864 weight_movement_bytes=352321536 total_nvlink_bytes=2229862400 weight_to_token_ratio=0.187651 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228291683e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=937934848 dispatch_meta_bytes=458648 combine_payload_bytes=937934848 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=938393496 ingress_bytes=1642577920 link_bytes=1642577920 token_bytes=1876328344 weight_movement_bytes=704643072 total_nvlink_bytes=2580971416 weight_to_token_ratio=0.375544 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.197750706e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129866 payload_rows_remote=113488 dup_rows=1206 dup_rows_remote=198 dedup_ratio=0.009201 router_tokens_per_expert=[0, 8178, 8184, 5, 0, 0, 14, 8099, 8, 2, 0, 8183, 0, 8, 0, 86, 4, 3, 0, 25, 0, 7, 8184, 7, 8176, 8, 3, 6, 8180, 0, 8164, 2] payload_rows_per_dst=[16378, 16226, 16348, 15600, 16384, 16340, 16366, 16224] dup_rows_per_dst=[1008, 108, 10, 12, 40, 14, 14, 0] combine_rows_per_src=[16378, 16226, 16348, 15600, 16384, 16340, 16366, 16224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131042 payload_rows_remote=114658 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000229 router_tokens_per_expert=[0, 8192, 8177, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16354, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 30, 0, 0, 0, 0] combine_rows_per_src=[16354, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938721280 dispatch_meta_bytes=458368 combine_payload_bytes=938721280 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939179648 ingress_bytes=1291042816 link_bytes=1291042816 token_bytes=1877900928 weight_movement_bytes=352321536 total_nvlink_bytes=2230222464 weight_to_token_ratio=0.187615 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229300862e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130362 payload_rows_remote=114004 dup_rows=710 dup_rows_remote=526 dedup_ratio=0.005417 router_tokens_per_expert=[0, 8179, 8106, 5, 0, 0, 13, 8128, 10, 2, 4, 8181, 0, 8, 0, 106, 6, 4, 0, 34, 0, 9, 8181, 9, 8174, 7, 4, 10, 8173, 0, 8177, 6] payload_rows_per_dst=[15964, 16282, 16368, 16372, 16276, 16372, 16370, 16358] dup_rows_per_dst=[0, 190, 10, 214, 58, 34, 20, 184] combine_rows_per_src=[15964, 16282, 16368, 16372, 16276, 16372, 16370, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131016 payload_rows_remote=114632 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.000427 router_tokens_per_expert=[0, 8192, 8167, 0, 0, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16334, 16378, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 56, 0, 0, 0, 0] combine_rows_per_src=[16334, 16378, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=929693696 dispatch_meta_bytes=454744 combine_payload_bytes=929693696 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=930148440 ingress_bytes=1986658304 link_bytes=1986658304 token_bytes=1859842136 weight_movement_bytes=1056964608 total_nvlink_bytes=2916806744 weight_to_token_ratio=0.568309 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.112445723e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127084 payload_rows_remote=110700 dup_rows=3988 dup_rows_remote=194 dedup_ratio=0.030426 router_tokens_per_expert=[1, 1, 0, 0, 1, 0, 6378, 0, 8191, 0, 8026, 1, 33, 0, 1, 8190, 1890, 1, 0, 7, 8176, 0, 1, 8191, 8189, 2, 1, 8, 8190, 0, 56, 1] payload_rows_per_dst=[16384, 12758, 16054, 16384, 16384, 16354, 16384, 16382] dup_rows_per_dst=[2, 0, 0, 64, 3794, 0, 16, 112] combine_rows_per_src=[16384, 12758, 16054, 16384, 16384, 16354, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=458752 combine_payload_bytes=939278336 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=939737088 ingress_bytes=1115439104 link_bytes=1115439104 token_bytes=1879015424 weight_movement_bytes=176160768 total_nvlink_bytes=2055176192 weight_to_token_ratio=0.093752 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245993375e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122118 payload_rows_remote=105754 dup_rows=8954 dup_rows_remote=44 dedup_ratio=0.068314 router_tokens_per_expert=[0, 0, 0, 0, 0, 4455, 8192, 8182, 0, 0, 22, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 3725, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16364, 16384, 16384, 16384, 7450, 16384, 16384] dup_rows_per_dst=[0, 8910, 44, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16364, 16384, 16384, 16384, 7450, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122792 payload_rows_remote=106408 dup_rows=8280 dup_rows_remote=8192 dedup_ratio=0.063171 router_tokens_per_expert=[3, 4, 4, 3, 5, 5545, 8186, 8188, 4, 2, 37, 6, 2, 3, 8183, 0, 2, 8187, 0, 8186, 0, 0, 0, 2607, 0, 0, 1, 8189, 1, 0, 2, 8186] payload_rows_per_dst=[16384, 15878, 16384, 16372, 16376, 8640, 16380, 16378] dup_rows_per_dst=[16, 7670, 88, 4, 0, 502, 0, 0] combine_rows_per_src=[16384, 15878, 16384, 16372, 16376, 8640, 16380, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=933920768 dispatch_meta_bytes=458120 combine_payload_bytes=933920768 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=934378888 ingress_bytes=1814724608 link_bytes=1814724608 token_bytes=1868299656 weight_movement_bytes=880803840 total_nvlink_bytes=2749103496 weight_to_token_ratio=0.471447 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.016170030e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939065344 dispatch_meta_bytes=458752 combine_payload_bytes=939065344 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939524096 ingress_bytes=1643708416 link_bytes=1643708416 token_bytes=1878589440 weight_movement_bytes=704643072 total_nvlink_bytes=2583232512 weight_to_token_ratio=0.375092 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204081012e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=906854400 dispatch_meta_bytes=443576 combine_payload_bytes=906854400 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=907297976 ingress_bytes=1259175936 link_bytes=1259175936 token_bytes=1814152376 weight_movement_bytes=352321536 total_nvlink_bytes=2166473912 weight_to_token_ratio=0.194207 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=29696 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.050859636e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120716 payload_rows_remote=104332 dup_rows=10356 dup_rows_remote=10356 dedup_ratio=0.079010 router_tokens_per_expert=[0, 0, 0, 0, 0, 4699, 8192, 8183, 0, 0, 12, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 3490, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 11502, 16384, 16384, 16384, 10910, 16384, 16384] dup_rows_per_dst=[0, 5474, 24, 0, 0, 4858, 0, 0] combine_rows_per_src=[16384, 11502, 16384, 16384, 16384, 10910, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=866336768 dispatch_meta_bytes=423192 combine_payload_bytes=866336768 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=866759960 ingress_bytes=1923301376 link_bytes=1923301376 token_bytes=1733096728 weight_movement_bytes=1056964608 total_nvlink_bytes=2790061336 weight_to_token_ratio=0.609871 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=22400 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.076968488e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=871694336 dispatch_meta_bytes=458400 combine_payload_bytes=871694336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=872152736 ingress_bytes=1224015872 link_bytes=1224015872 token_bytes=1743847072 weight_movement_bytes=352321536 total_nvlink_bytes=2096168608 weight_to_token_ratio=0.202037 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.853977954e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121848 payload_rows_remote=105464 dup_rows=9224 dup_rows_remote=9178 dedup_ratio=0.070374 router_tokens_per_expert=[10, 1, 17, 8, 6, 4333, 8179, 8185, 7, 7, 45, 0, 8, 3, 8164, 0, 0, 8179, 1, 8177, 0, 0, 0, 3813, 0, 0, 4, 8185, 11, 0, 12, 8181] payload_rows_per_dst=[16384, 16370, 16382, 16344, 15994, 7626, 16378, 16370] dup_rows_per_dst=[46, 8974, 94, 78, 0, 0, 12, 20] combine_rows_per_src=[16384, 16370, 16382, 16344, 15994, 7626, 16378, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125030 payload_rows_remote=108646 dup_rows=6042 dup_rows_remote=6042 dedup_ratio=0.046097 router_tokens_per_expert=[0, 0, 0, 0, 0, 5171, 8192, 8192, 0, 0, 11, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 3010, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 10342, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 22, 0, 0, 6020, 0, 0] combine_rows_per_src=[16384, 10342, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=854687744 dispatch_meta_bytes=458752 combine_payload_bytes=854687744 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=855146496 ingress_bytes=1207009280 link_bytes=1207009280 token_bytes=1709834240 weight_movement_bytes=352321536 total_nvlink_bytes=2062155776 weight_to_token_ratio=0.206056 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.758748138e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120640 payload_rows_remote=104264 dup_rows=10432 dup_rows_remote=10424 dedup_ratio=0.079590 router_tokens_per_expert=[16, 0, 14, 9, 21, 3266, 8171, 8180, 12, 9, 33, 13, 6, 8, 8164, 0, 0, 8178, 0, 8175, 0, 0, 0, 4880, 0, 3, 4, 8184, 13, 0, 0, 8177] payload_rows_per_dst=[16282, 6606, 15938, 16344, 16350, 16368, 16376, 16376] dup_rows_per_dst=[36, 70, 106, 12, 436, 9752, 12, 8] combine_rows_per_src=[16282, 6606, 15938, 16344, 16350, 16368, 16376, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=863961088 dispatch_meta_bytes=458568 combine_payload_bytes=863961088 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=864419656 ingress_bytes=1216282624 link_bytes=1216282624 token_bytes=1728380744 weight_movement_bytes=352321536 total_nvlink_bytes=2080702280 weight_to_token_ratio=0.203845 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.810674993e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125496 payload_rows_remote=109112 dup_rows=5576 dup_rows_remote=0 dedup_ratio=0.042542 router_tokens_per_expert=[0, 0, 0, 0, 0, 5404, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 2788, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 10808, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 5576, 0, 0] combine_rows_per_src=[16384, 10808, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=890028032 dispatch_meta_bytes=458752 combine_payload_bytes=890028032 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=890486784 ingress_bytes=1242349568 link_bytes=1242349568 token_bytes=1780514816 weight_movement_bytes=352321536 total_nvlink_bytes=2132836352 weight_to_token_ratio=0.197876 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.956638999e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130920 payload_rows_remote=114536 dup_rows=152 dup_rows_remote=146 dedup_ratio=0.001160 router_tokens_per_expert=[0, 8191, 8125, 1, 0, 0, 2, 8184, 0, 0, 3, 8191, 0, 0, 0, 64, 1, 0, 0, 3, 0, 1, 8191, 1, 8190, 1, 1, 1, 8191, 0, 8190, 4] payload_rows_per_dst=[16252, 16372, 16382, 16382, 16384, 16384, 16384, 16380] dup_rows_per_dst=[0, 0, 6, 128, 6, 2, 2, 8] combine_rows_per_src=[16252, 16372, 16382, 16382, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=854130688 dispatch_meta_bytes=458752 combine_payload_bytes=854130688 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=854589440 ingress_bytes=1382612992 link_bytes=1382612992 token_bytes=1708720128 weight_movement_bytes=528482304 total_nvlink_bytes=2237202432 weight_to_token_ratio=0.309285 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.742055624e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130890 payload_rows_remote=114506 dup_rows=182 dup_rows_remote=182 dedup_ratio=0.001389 router_tokens_per_expert=[0, 0, 6, 0, 0, 0, 8192, 0, 0, 0, 8119, 8179, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 71, 0, 8192, 8185, 0, 0, 1, 15, 8192, 0, 8192] payload_rows_per_dst=[16362, 16384, 16238, 16384, 16384, 16384, 16370, 16384] dup_rows_per_dst=[8, 0, 0, 0, 0, 142, 2, 30] combine_rows_per_src=[16362, 16384, 16238, 16384, 16384, 16384, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130928 payload_rows_remote=114648 dup_rows=144 dup_rows_remote=140 dedup_ratio=0.001099 router_tokens_per_expert=[3, 0, 0, 4, 4, 0, 8188, 4, 1, 5, 8136, 8187, 4, 3, 8187, 0, 0, 0, 0, 5, 0, 49, 0, 8188, 8183, 1, 3, 0, 11, 8188, 0, 8182] payload_rows_per_dst=[16384, 16384, 16280, 16384, 16384, 16376, 16372, 16364] dup_rows_per_dst=[4, 8, 4, 4, 2, 98, 2, 22] combine_rows_per_src=[16384, 16384, 16280, 16384, 16384, 16376, 16372, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=893845504 dispatch_meta_bytes=436448 combine_payload_bytes=893845504 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=894281952 ingress_bytes=1246167040 link_bytes=1246167040 token_bytes=1788127456 weight_movement_bytes=352321536 total_nvlink_bytes=2140448992 weight_to_token_ratio=0.197034 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.978015248e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130982 payload_rows_remote=114598 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.000687 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8163, 8176, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 32, 0, 8192, 8192, 0, 0, 0, 13, 8192, 0, 8192] payload_rows_per_dst=[16352, 16384, 16326, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 64, 0, 26] combine_rows_per_src=[16352, 16384, 16326, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[3, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938278912 dispatch_meta_bytes=458728 combine_payload_bytes=938278912 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938737640 ingress_bytes=1290600448 link_bytes=1290600448 token_bytes=1877016552 weight_movement_bytes=352321536 total_nvlink_bytes=2229338088 weight_to_token_ratio=0.187703 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226823786e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938033152 dispatch_meta_bytes=458752 combine_payload_bytes=938033152 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=938491904 ingress_bytes=1818836992 link_bytes=1818836992 token_bytes=1876525056 weight_movement_bytes=880803840 total_nvlink_bytes=2757328896 weight_to_token_ratio=0.469380 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018472794e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130440 payload_rows_remote=114058 dup_rows=632 dup_rows_remote=606 dedup_ratio=0.004822 router_tokens_per_expert=[3, 0, 0, 8, 9, 0, 8185, 10, 9, 13, 8171, 8181, 4, 3, 8180, 0, 0, 0, 0, 11, 0, 0, 0, 8184, 8179, 4, 7, 4, 12, 8184, 0, 8175] payload_rows_per_dst=[16382, 16384, 16382, 16374, 16384, 15872, 16368, 16294] dup_rows_per_dst=[26, 58, 476, 4, 6, 0, 60, 2] combine_rows_per_src=[16382, 16384, 16382, 16374, 16384, 15872, 16368, 16294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=459152 combine_payload_bytes=939196416 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939655568 ingress_bytes=1820000256 link_bytes=1820000256 token_bytes=1878851984 weight_movement_bytes=880803840 total_nvlink_bytes=2759655824 weight_to_token_ratio=0.468799 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019124173e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131052 payload_rows_remote=114668 dup_rows=20 dup_rows_remote=16 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8182, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 8192, 0, 0, 2, 2, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16364, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 12, 4, 4] combine_rows_per_src=[16384, 16384, 16364, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938786816 dispatch_meta_bytes=458752 combine_payload_bytes=938786816 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939245568 ingress_bytes=1291108352 link_bytes=1291108352 token_bytes=1878032384 weight_movement_bytes=352321536 total_nvlink_bytes=2230353920 weight_to_token_ratio=0.187601 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229667836e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130764 payload_rows_remote=114422 dup_rows=308 dup_rows_remote=248 dedup_ratio=0.002350 router_tokens_per_expert=[0, 0, 5, 11, 11, 0, 8184, 9, 9, 14, 8137, 8171, 2, 5, 8178, 5, 0, 0, 0, 13, 0, 36, 0, 8177, 8168, 2, 8, 22, 15, 8183, 10, 8161] payload_rows_per_dst=[16358, 16292, 16304, 16374, 16370, 16362, 16362, 16342] dup_rows_per_dst=[0, 24, 32, 14, 8, 144, 26, 60] combine_rows_per_src=[16358, 16292, 16304, 16374, 16370, 16362, 16362, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=934363136 dispatch_meta_bytes=458656 combine_payload_bytes=934363136 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=934821792 ingress_bytes=1462845440 link_bytes=1462845440 token_bytes=1869184928 weight_movement_bytes=528482304 total_nvlink_bytes=2397667232 weight_to_token_ratio=0.282734 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.191323842e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130998 payload_rows_remote=114614 dup_rows=74 dup_rows_remote=8 dedup_ratio=0.000565 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8155, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 33, 0, 8192, 8192, 0, 0, 0, 4, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16310, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 66, 0, 8] combine_rows_per_src=[16384, 16384, 16310, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458736 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939818992 ingress_bytes=1467842560 link_bytes=1467842560 token_bytes=1879179248 weight_movement_bytes=528482304 total_nvlink_bytes=2407661552 weight_to_token_ratio=0.281230 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219305628e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=937345024 dispatch_meta_bytes=458680 combine_payload_bytes=937345024 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=937803704 ingress_bytes=1994309632 link_bytes=1994309632 token_bytes=1875148728 weight_movement_bytes=1056964608 total_nvlink_bytes=2932113336 weight_to_token_ratio=0.563670 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.116730147e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131048 payload_rows_remote=114676 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 1, 8192, 8182, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 11, 0, 0] payload_rows_per_dst=[16384, 16372, 16384, 16380, 16376, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 0, 22] combine_rows_per_src=[16384, 16372, 16384, 16380, 16376, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126776 payload_rows_remote=110396 dup_rows=4296 dup_rows_remote=4296 dedup_ratio=0.032776 router_tokens_per_expert=[1, 0, 1, 1, 0, 6052, 8191, 8186, 1, 1, 33, 1, 1, 1, 8189, 0, 0, 8191, 0, 8190, 1, 0, 1, 2113, 0, 0, 0, 8189, 1, 0, 0, 8191] payload_rows_per_dst=[16384, 12104, 16384, 16380, 16380, 16382, 16378, 16384] dup_rows_per_dst=[4, 0, 70, 2, 0, 4220, 0, 0] combine_rows_per_src=[16384, 12104, 16384, 16380, 16380, 16382, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131016 payload_rows_remote=114632 dup_rows=56 dup_rows_remote=44 dedup_ratio=0.000427 router_tokens_per_expert=[0, 2, 0, 8188, 0, 3, 0, 0, 2, 8188, 4, 4, 8186, 5, 4, 0, 16, 8185, 8187, 5, 0, 8187, 0, 0, 0, 8174, 4, 8188, 4, 0, 0, 0] payload_rows_per_dst=[16376, 16374, 16384, 16384, 16384, 16374, 16356, 16384] dup_rows_per_dst=[4, 0, 12, 6, 34, 0, 0, 0] combine_rows_per_src=[16376, 16374, 16384, 16384, 16384, 16374, 16356, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 2, 2, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131034 payload_rows_remote=114658 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000290 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 8177, 0, 8192, 0, 19, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16384, 16384, 16354, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 38] combine_rows_per_src=[16384, 16384, 16384, 16376, 16384, 16384, 16354, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938917888 dispatch_meta_bytes=458488 combine_payload_bytes=938917888 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939376376 ingress_bytes=1643560960 link_bytes=1643560960 token_bytes=1878294264 weight_movement_bytes=704643072 total_nvlink_bytes=2582937336 weight_to_token_ratio=0.375151 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203255320e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458800 combine_payload_bytes=939425792 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=939884592 ingress_bytes=1644068864 link_bytes=1644068864 token_bytes=1879310384 weight_movement_bytes=704643072 total_nvlink_bytes=2583953456 weight_to_token_ratio=0.374948 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206099370e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=904364032 dispatch_meta_bytes=458768 combine_payload_bytes=904364032 prefetch_bytes=176160768 reduce_bytes=1409286144 egress_bytes=904822800 ingress_bytes=2489810944 link_bytes=2489810944 token_bytes=1809186832 weight_movement_bytes=1585446912 total_nvlink_bytes=3394633744 weight_to_token_ratio=0.876331 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.394190199e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130846 payload_rows_remote=114472 dup_rows=226 dup_rows_remote=210 dedup_ratio=0.001724 router_tokens_per_expert=[5, 0, 0, 8183, 0, 8, 0, 0, 8, 8182, 11, 10, 8182, 16, 5, 0, 0, 8181, 8184, 11, 1, 8181, 0, 8, 0, 8160, 9, 8181, 5, 2, 3, 0] payload_rows_per_dst=[16374, 16378, 16384, 16382, 16344, 16340, 16368, 16276] dup_rows_per_dst=[16, 32, 38, 48, 0, 16, 76, 0] combine_rows_per_src=[16374, 16378, 16384, 16382, 16344, 16340, 16368, 16276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939065344 dispatch_meta_bytes=458704 combine_payload_bytes=939065344 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=939524048 ingress_bytes=1643708416 link_bytes=1643708416 token_bytes=1878589392 weight_movement_bytes=704643072 total_nvlink_bytes=2583232464 weight_to_token_ratio=0.375092 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204081012e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=458784 combine_payload_bytes=939278336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939737120 ingress_bytes=1291599872 link_bytes=1291599872 token_bytes=1879015456 weight_movement_bytes=352321536 total_nvlink_bytes=2231336992 weight_to_token_ratio=0.187503 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232420143e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130836 payload_rows_remote=114454 dup_rows=236 dup_rows_remote=208 dedup_ratio=0.001801 router_tokens_per_expert=[9, 0, 0, 8181, 0, 8, 0, 4, 7, 8183, 12, 11, 8165, 9, 3, 0, 11, 8171, 8180, 12, 2, 8177, 0, 25, 0, 8151, 10, 8182, 9, 14, 0, 0] payload_rows_per_dst=[16374, 16384, 16290, 16348, 16372, 16364, 16322, 16382] dup_rows_per_dst=[6, 8, 42, 6, 22, 50, 74, 28] combine_rows_per_src=[16374, 16384, 16290, 16348, 16372, 16364, 16322, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937754624 dispatch_meta_bytes=458728 combine_payload_bytes=937754624 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938213352 ingress_bytes=1290076160 link_bytes=1290076160 token_bytes=1875967976 weight_movement_bytes=352321536 total_nvlink_bytes=2228289512 weight_to_token_ratio=0.187808 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223887992e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939982848 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879506944 weight_movement_bytes=704643072 total_nvlink_bytes=2584150016 weight_to_token_ratio=0.374908 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937607168 dispatch_meta_bytes=458648 combine_payload_bytes=937607168 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938065816 ingress_bytes=1289928704 link_bytes=1289928704 token_bytes=1875672984 weight_movement_bytes=352321536 total_nvlink_bytes=2227994520 weight_to_token_ratio=0.187837 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223062300e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114682 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000046 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 8189, 0, 8192, 0, 3, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16378, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114678 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[0, 8192, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16376, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 10, 0, 0] combine_rows_per_src=[16376, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131002 payload_rows_remote=114620 dup_rows=70 dup_rows_remote=70 dedup_ratio=0.000534 router_tokens_per_expert=[0, 0, 22, 1, 1, 0, 8191, 1, 2, 1, 8175, 8191, 0, 1, 8190, 0, 0, 0, 0, 1, 0, 9, 0, 8191, 8176, 0, 1, 0, 1, 8190, 0, 8191] payload_rows_per_dst=[16384, 16384, 16352, 16382, 16382, 16382, 16354, 16382] dup_rows_per_dst=[44, 2, 4, 0, 0, 18, 0, 2] combine_rows_per_src=[16384, 16384, 16352, 16382, 16382, 16382, 16354, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131024 payload_rows_remote=114648 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000366 router_tokens_per_expert=[0, 8188, 8187, 0, 6, 6, 0, 8188, 0, 0, 0, 0, 3, 0, 4, 5, 8183, 4, 4, 0, 0, 5, 0, 3, 0, 8184, 8188, 4, 0, 0, 8188, 8186] payload_rows_per_dst=[16374, 16384, 16376, 16384, 16374, 16384, 16376, 16372] dup_rows_per_dst=[0, 16, 0, 16, 8, 8, 0, 0] combine_rows_per_src=[16374, 16384, 16376, 16384, 16374, 16384, 16376, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=458752 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939933696 ingress_bytes=1467957248 link_bytes=1467957248 token_bytes=1879408640 weight_movement_bytes=528482304 total_nvlink_bytes=2407890944 weight_to_token_ratio=0.281196 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219947833e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458752 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939900928 ingress_bytes=1820246016 link_bytes=1820246016 token_bytes=1879343104 weight_movement_bytes=880803840 total_nvlink_bytes=2760146944 weight_to_token_ratio=0.468676 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019261788e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130876 payload_rows_remote=114510 dup_rows=196 dup_rows_remote=106 dedup_ratio=0.001495 router_tokens_per_expert=[0, 8182, 8180, 0, 19, 13, 2, 8183, 0, 0, 0, 0, 6, 0, 9, 14, 8177, 9, 10, 0, 0, 12, 0, 0, 0, 8167, 8183, 8, 0, 1, 8182, 8179] payload_rows_per_dst=[16366, 16346, 16364, 16384, 16374, 16360, 16348, 16334] dup_rows_per_dst=[90, 18, 24, 40, 18, 4, 0, 2] combine_rows_per_src=[16366, 16346, 16364, 16384, 16374, 16360, 16348, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938967040 dispatch_meta_bytes=458760 combine_payload_bytes=938967040 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939425800 ingress_bytes=1291288576 link_bytes=1291288576 token_bytes=1878392840 weight_movement_bytes=352321536 total_nvlink_bytes=2230714376 weight_to_token_ratio=0.187565 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230677015e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458784 combine_payload_bytes=939196416 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=939655200 ingress_bytes=1643839488 link_bytes=1643839488 token_bytes=1878851616 weight_movement_bytes=704643072 total_nvlink_bytes=2583494688 weight_to_token_ratio=0.375039 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204814960e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130708 payload_rows_remote=114354 dup_rows=364 dup_rows_remote=270 dedup_ratio=0.002777 router_tokens_per_expert=[0, 8182, 8181, 0, 29, 16, 2, 8175, 0, 0, 1, 0, 10, 0, 8, 12, 8172, 9, 11, 0, 4, 15, 0, 0, 0, 8157, 8182, 10, 0, 4, 8180, 8176] payload_rows_per_dst=[16334, 16318, 16360, 16284, 16366, 16358, 16334, 16354] dup_rows_per_dst=[0, 68, 30, 40, 18, 18, 96, 94] combine_rows_per_src=[16334, 16318, 16360, 16284, 16366, 16358, 16334, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879506944 weight_movement_bytes=352321536 total_nvlink_bytes=2231828480 weight_to_token_ratio=0.187454 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29116178728 prefetch_bytes=5813305344 reduce_bytes=5108662272 weight_movement_bytes=10921967616 egress_bytes=14561730856 ingress_bytes=25476415488 link_bytes=25476415488 total_nvlink_bytes=40038146344 weight_to_token_ratio=0.375117 prefetch_share_of_ingress=0.228184 dedup_ratio=0.029503 prefetch_slots=66 unused_prefetch_slots=190 slot_utilization=0.257812 reduce_slots=29 remote_demand=33 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018111 gemm_inner_pad_upper_bound=21844 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.916080772e-05 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938065920 dispatch_meta_bytes=458464 combine_payload_bytes=938065920 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=938524384 ingress_bytes=1818869760 link_bytes=1818869760 token_bytes=1876590304 weight_movement_bytes=880803840 total_nvlink_bytes=2757394144 weight_to_token_ratio=0.469364 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018491142e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29167515952 prefetch_bytes=4404019200 reduce_bytes=5813305344 weight_movement_bytes=10217324544 egress_bytes=14587427120 ingress_bytes=24797413376 link_bytes=24797413376 total_nvlink_bytes=39384840496 weight_to_token_ratio=0.350298 prefetch_share_of_ingress=0.177600 dedup_ratio=0.029970 prefetch_slots=50 unused_prefetch_slots=206 slot_utilization=0.195312 reduce_slots=33 remote_demand=25 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019650 gemm_inner_pad_upper_bound=20701 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.678447747e-05 prefetch_bytes_per_gemm_flop=1.541291824e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939982848 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879506944 weight_movement_bytes=880803840 total_nvlink_bytes=2760310784 weight_to_token_ratio=0.468636 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=936787968 dispatch_meta_bytes=458496 combine_payload_bytes=936787968 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=937246464 ingress_bytes=1817591808 link_bytes=1817591808 token_bytes=1874034432 weight_movement_bytes=880803840 total_nvlink_bytes=2754838272 weight_to_token_ratio=0.470004 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.017775542e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29309272128 prefetch_bytes=5637144576 reduce_bytes=1937768448 weight_movement_bytes=7574913024 egress_bytes=14658306112 ingress_bytes=22225879040 link_bytes=22225879040 total_nvlink_bytes=36884185152 weight_to_token_ratio=0.258448 prefetch_share_of_ingress=0.253630 dedup_ratio=0.024417 prefetch_slots=64 unused_prefetch_slots=192 slot_utilization=0.250000 reduce_slots=11 remote_demand=32 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=20828 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.778477818e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28129917536 prefetch_bytes=5460983808 reduce_bytes=5284823040 weight_movement_bytes=10745806848 egress_bytes=14068578912 ingress_bytes=24807145472 link_bytes=24807145472 total_nvlink_bytes=38875724384 weight_to_token_ratio=0.382006 prefetch_share_of_ingress=0.220138 dedup_ratio=0.057419 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=30 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019176 gemm_inner_pad_upper_bound=20955 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.681853727e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28761204664 prefetch_bytes=4404019200 reduce_bytes=7927234560 weight_movement_bytes=12331253760 egress_bytes=14384244664 ingress_bytes=26708213760 link_bytes=26708213760 total_nvlink_bytes=41092458424 weight_to_token_ratio=0.428746 prefetch_share_of_ingress=0.164894 dedup_ratio=0.039055 prefetch_slots=50 unused_prefetch_slots=206 slot_utilization=0.195312 reduce_slots=45 remote_demand=25 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020478 gemm_inner_pad_upper_bound=19431 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.347178031e-05 prefetch_bytes_per_gemm_flop=1.541291824e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114654 dup_rows=34 dup_rows_remote=32 dedup_ratio=0.000259 router_tokens_per_expert=[0, 2, 0, 8191, 0, 1, 0, 0, 1, 8190, 1, 1, 8187, 0, 0, 0, 1, 8190, 8186, 1, 1, 8191, 0, 4, 0, 8187, 1, 8191, 1, 8, 0, 0] payload_rows_per_dst=[16382, 16372, 16382, 16374, 16384, 16384, 16376, 16384] dup_rows_per_dst=[4, 0, 4, 0, 2, 8, 0, 16] combine_rows_per_src=[16382, 16372, 16382, 16374, 16384, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29261462808 prefetch_bytes=4227858432 reduce_bytes=3699376128 weight_movement_bytes=7927234560 egress_bytes=14634384664 ingress_bytes=22554312704 link_bytes=22554312704 total_nvlink_bytes=37188697368 weight_to_token_ratio=0.270910 prefetch_share_of_ingress=0.187452 dedup_ratio=0.024924 prefetch_slots=48 unused_prefetch_slots=208 slot_utilization=0.187500 reduce_slots=21 remote_demand=24 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018584 gemm_inner_pad_upper_bound=21209 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.893421032e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879506944 weight_movement_bytes=352321536 total_nvlink_bytes=2231828480 weight_to_token_ratio=0.187454 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 4, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=458744 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=939704312 ingress_bytes=2172370944 link_bytes=2172370944 token_bytes=1878949880 weight_movement_bytes=1233125376 total_nvlink_bytes=3112075256 weight_to_token_ratio=0.656284 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216437049e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29236930240 prefetch_bytes=5108662272 reduce_bytes=4404019200 weight_movement_bytes=9512681472 egress_bytes=14622123712 ingress_bytes=24127488000 link_bytes=24127488000 total_nvlink_bytes=38749611712 weight_to_token_ratio=0.325365 prefetch_share_of_ingress=0.211736 dedup_ratio=0.026479 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=25 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020005 gemm_inner_pad_upper_bound=19558 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.443991343e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131052 payload_rows_remote=114674 dup_rows=20 dup_rows_remote=18 dedup_ratio=0.000153 router_tokens_per_expert=[1, 8191, 8189, 0, 1, 1, 0, 8191, 0, 0, 0, 0, 1, 0, 1, 1, 8188, 1, 1, 0, 4, 2, 0, 0, 0, 8190, 8191, 1, 0, 0, 8191, 8190] payload_rows_per_dst=[16378, 16384, 16382, 16384, 16378, 16384, 16382, 16380] dup_rows_per_dst=[2, 2, 0, 4, 2, 10, 0, 0] combine_rows_per_src=[16378, 16384, 16382, 16384, 16378, 16384, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=458768 combine_payload_bytes=939409408 prefetch_bytes=0 reduce_bytes=0 egress_bytes=939868176 ingress_bytes=939409408 link_bytes=939868176 token_bytes=1879277584 weight_movement_bytes=0 total_nvlink_bytes=1879277584 weight_to_token_ratio=0.000000 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.262869465e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29344249984 prefetch_bytes=5284823040 reduce_bytes=6165626880 weight_movement_bytes=11450449920 egress_bytes=14675785856 ingress_bytes=26118914048 link_bytes=26119372816 total_nvlink_bytes=40794699904 weight_to_token_ratio=0.390211 prefetch_share_of_ingress=0.202337 dedup_ratio=0.023202 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=35 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019176 gemm_inner_pad_upper_bound=20701 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.141099063e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none - [2026-09-16 21:37:06] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 567.6 | throughput per GPU (TFLOP/s/GPU): 587.4 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.047690E+01 | loss scale: 1.0 | grad norm: 38.863 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=102160 payload_rows_remote=93416 dup_rows=28912 dup_rows_remote=27602 dedup_ratio=0.220581 router_tokens_per_expert=[9, 7, 1507, 4, 304, 1, 4490, 7, 16, 7903, 2, 2128, 1055, 5132, 6, 7697, 6786, 7465, 2581, 2, 15, 3618, 964, 7, 9, 3705, 2327, 3667, 1022, 78, 2903, 119] payload_rows_per_dst=[15336, 15028, 16014, 8744, 14442, 11448, 13206, 7942] dup_rows_per_dst=[2648, 7400, 4084, 1310, 4388, 2900, 5282, 900] combine_rows_per_src=[15336, 15028, 16014, 8744, 14442, 11448, 13206, 7942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=100656 payload_rows_remote=85188 dup_rows=30416 dup_rows_remote=27268 dedup_ratio=0.232056 router_tokens_per_expert=[29, 10, 1808, 3, 438, 0, 4023, 11, 4, 7490, 6, 1300, 1372, 4848, 2, 6919, 5237, 7458, 5032, 3, 7, 3648, 795, 1, 2, 3760, 1994, 3221, 2048, 82, 3788, 197] payload_rows_per_dst=[15468, 14438, 15836, 13244, 12700, 8552, 9984, 10434] dup_rows_per_dst=[3148, 5694, 7586, 4952, 5466, 350, 1424, 1796] combine_rows_per_src=[15468, 14438, 15836, 13244, 12700, 8552, 9984, 10434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=100936 payload_rows_remote=84966 dup_rows=30136 dup_rows_remote=24618 dedup_ratio=0.229919 router_tokens_per_expert=[24, 7, 1974, 1, 222, 1, 4736, 3, 27, 7869, 4, 2844, 1711, 5902, 0, 7262, 4804, 7405, 2371, 3, 8, 3793, 644, 7, 3, 3600, 2027, 3101, 1370, 88, 3533, 192] payload_rows_per_dst=[15384, 15554, 15970, 12094, 12862, 9986, 9768, 9318] dup_rows_per_dst=[3438, 7704, 5518, 4696, 4520, 1720, 1492, 1048] combine_rows_per_src=[15384, 15554, 15970, 12094, 12862, 9986, 9768, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96788 payload_rows_remote=82972 dup_rows=34284 dup_rows_remote=27914 dedup_ratio=0.261566 router_tokens_per_expert=[16, 0, 1961, 20, 336, 4, 3808, 1, 13, 7749, 8, 1995, 2211, 4685, 3, 7423, 6176, 6958, 3363, 1, 1, 3189, 816, 7, 0, 3024, 2921, 4148, 1392, 63, 3085, 159] payload_rows_per_dst=[14664, 8120, 5978, 14188, 10396, 14004, 13816, 15622] dup_rows_per_dst=[3230, 178, 732, 8218, 1758, 5416, 6370, 8382] combine_rows_per_src=[14664, 8120, 5978, 14188, 10396, 14004, 13816, 15622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98372 payload_rows_remote=84016 dup_rows=32700 dup_rows_remote=28006 dedup_ratio=0.249481 router_tokens_per_expert=[5, 0, 1946, 0, 239, 0, 3031, 1, 2, 7843, 0, 2127, 1584, 5452, 0, 7269, 6495, 7222, 3831, 0, 0, 3345, 761, 1, 1, 2678, 2820, 3771, 1420, 85, 3359, 248] payload_rows_per_dst=[15096, 14356, 16022, 13892, 12160, 7944, 9586, 9316] dup_rows_per_dst=[3250, 4694, 9090, 5742, 7334, 270, 1412, 908] combine_rows_per_src=[15096, 14356, 16022, 13892, 12160, 7944, 9586, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=89466 payload_rows_remote=73872 dup_rows=41606 dup_rows_remote=32984 dedup_ratio=0.317429 router_tokens_per_expert=[5, 1, 1468, 5, 151, 2, 4527, 0, 4, 7896, 1, 1904, 1046, 5008, 1, 7665, 6584, 7601, 2565, 0, 3, 3884, 1026, 1, 0, 3340, 2761, 3335, 1040, 128, 3427, 157] payload_rows_per_dst=[3256, 9220, 3816, 13114, 15842, 15150, 13474, 15594] dup_rows_per_dst=[152, 140, 2, 8362, 10476, 7728, 6124, 8622] combine_rows_per_src=[3256, 9220, 3816, 13114, 15842, 15150, 13474, 15594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=100556 payload_rows_remote=85294 dup_rows=30516 dup_rows_remote=22360 dedup_ratio=0.232819 router_tokens_per_expert=[5, 3, 1475, 2, 224, 0, 4831, 5, 5, 7865, 0, 2270, 1240, 4991, 1, 7349, 5984, 7634, 1780, 2, 2, 4838, 771, 1, 6, 3233, 3029, 3361, 1251, 123, 3083, 172] payload_rows_per_dst=[15624, 11320, 10704, 11110, 7680, 15262, 13310, 15546] dup_rows_per_dst=[2614, 1384, 1718, 1274, 1382, 8156, 5948, 8040] combine_rows_per_src=[15624, 11320, 10704, 11110, 7680, 15262, 13310, 15546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=765263872 dispatch_meta_bytes=484072 combine_payload_bytes=765263872 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=765747944 ingress_bytes=1822228480 link_bytes=1822228480 token_bytes=1531011816 weight_movement_bytes=1056964608 total_nvlink_bytes=2587976424 weight_to_token_ratio=0.690370 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=15104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.020371885e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=697860096 dispatch_meta_bytes=449824 combine_payload_bytes=697860096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=698309920 ingress_bytes=1050181632 link_bytes=1050181632 token_bytes=1396170016 weight_movement_bytes=352321536 total_nvlink_bytes=1748491552 weight_to_token_ratio=0.252349 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=25984 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.880578772e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=696041472 dispatch_meta_bytes=438336 combine_payload_bytes=696041472 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=696479808 ingress_bytes=1400684544 link_bytes=1400684544 token_bytes=1392521280 weight_movement_bytes=704643072 total_nvlink_bytes=2097164352 weight_to_token_ratio=0.506020 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=24832 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.843248772e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=679706624 dispatch_meta_bytes=443544 combine_payload_bytes=679706624 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=680150168 ingress_bytes=1384349696 link_bytes=1384349696 token_bytes=1359856792 weight_movement_bytes=704643072 total_nvlink_bytes=2064499864 weight_to_token_ratio=0.518174 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=13696 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.751780442e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=688259072 dispatch_meta_bytes=448088 combine_payload_bytes=688259072 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=688707160 ingress_bytes=1040580608 link_bytes=1040580608 token_bytes=1376966232 weight_movement_bytes=352321536 total_nvlink_bytes=1729287768 weight_to_token_ratio=0.255868 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=17408 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.826817045e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=605159424 dispatch_meta_bytes=427424 combine_payload_bytes=605159424 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=605586848 ingress_bytes=957480960 link_bytes=957480960 token_bytes=1210746272 weight_movement_bytes=352321536 total_nvlink_bytes=1563067808 weight_to_token_ratio=0.290995 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=13568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.361493704e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=698728448 dispatch_meta_bytes=430616 combine_payload_bytes=698728448 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=699159064 ingress_bytes=1051049984 link_bytes=1051049984 token_bytes=1397887512 weight_movement_bytes=352321536 total_nvlink_bytes=1750209048 weight_to_token_ratio=0.252039 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=15232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.885441181e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125434 payload_rows_remote=109050 dup_rows=5638 dup_rows_remote=5590 dedup_ratio=0.043015 router_tokens_per_expert=[8185, 0, 27, 1, 27, 3, 0, 4, 2210, 0, 9, 3, 14, 8188, 7, 7, 1371, 2, 1, 0, 0, 2, 0, 7975, 0, 8177, 6781, 6174, 0, 0, 8186, 8182] payload_rows_per_dst=[16374, 16368, 15260, 16384, 16380, 15954, 12348, 16366] dup_rows_per_dst=[52, 54, 2746, 48, 2738, 0, 0, 0] combine_rows_per_src=[16374, 16368, 15260, 16384, 16380, 15954, 12348, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110810 payload_rows_remote=94426 dup_rows=20262 dup_rows_remote=19192 dedup_ratio=0.154587 router_tokens_per_expert=[8191, 0, 514, 2, 5, 0, 0, 0, 1097, 0, 1, 0, 0, 8189, 6, 2, 2722, 0, 0, 0, 0, 1, 0, 7835, 0, 8184, 6046, 6357, 3, 6, 8184, 8191] payload_rows_per_dst=[16384, 16372, 13262, 16346, 16384, 15672, 16378, 12] dup_rows_per_dst=[1070, 6, 1026, 8, 5434, 0, 12712, 6] combine_rows_per_src=[16384, 16372, 13262, 16346, 16384, 15672, 16378, 12] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124930 payload_rows_remote=109812 dup_rows=6142 dup_rows_remote=4290 dedup_ratio=0.046860 router_tokens_per_expert=[8189, 2, 52, 1, 39, 0, 0, 0, 4162, 0, 0, 4, 1, 8184, 2, 0, 2057, 0, 2, 0, 1, 1, 0, 8159, 0, 8181, 4319, 5803, 1, 1, 8187, 8188] payload_rows_per_dst=[16384, 16368, 15118, 16372, 16382, 16320, 11606, 16380] dup_rows_per_dst=[104, 72, 1852, 2, 4108, 2, 0, 2] combine_rows_per_src=[16384, 16368, 15118, 16372, 16382, 16320, 11606, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114880 payload_rows_remote=104076 dup_rows=16192 dup_rows_remote=14438 dedup_ratio=0.123535 router_tokens_per_expert=[8187, 0, 625, 10, 4, 0, 0, 0, 2408, 12, 1, 0, 2, 8192, 3, 0, 1679, 6, 0, 3, 0, 0, 0, 8001, 0, 8180, 6902, 4949, 0, 4, 8186, 8182] payload_rows_per_dst=[16384, 16362, 13196, 16384, 9372, 16002, 10804, 16376] dup_rows_per_dst=[1260, 6, 2790, 10, 1518, 0, 1754, 8854] combine_rows_per_src=[16384, 16362, 13196, 16384, 9372, 16002, 10804, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116680 payload_rows_remote=100296 dup_rows=14392 dup_rows_remote=14362 dedup_ratio=0.109802 router_tokens_per_expert=[8192, 0, 222, 0, 15, 0, 0, 0, 4310, 1, 1, 0, 0, 8192, 0, 0, 2240, 0, 0, 1, 0, 0, 0, 8182, 0, 8192, 5584, 4026, 0, 0, 8190, 8188] payload_rows_per_dst=[16384, 16384, 14774, 16384, 16384, 16364, 13842, 6164] dup_rows_per_dst=[444, 30, 5018, 0, 4482, 0, 4418, 0] combine_rows_per_src=[16384, 16384, 14774, 16384, 16384, 16364, 13842, 6164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=103492 payload_rows_remote=87116 dup_rows=27580 dup_rows_remote=9812 dedup_ratio=0.210419 router_tokens_per_expert=[8188, 2, 25, 2, 14, 1, 0, 0, 3768, 2, 1, 1, 0, 8186, 4, 2, 953, 1, 1, 3, 2, 1, 2, 8168, 0, 8188, 5959, 5690, 0, 0, 8184, 8188] payload_rows_per_dst=[12924, 16222, 7862, 16380, 1910, 16342, 15476, 16376] dup_rows_per_dst=[48, 24, 360, 4, 6, 2066, 7304, 17768] combine_rows_per_src=[12924, 16222, 7862, 16380, 1910, 16342, 15476, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127530 payload_rows_remote=111186 dup_rows=3542 dup_rows_remote=3540 dedup_ratio=0.027023 router_tokens_per_expert=[8189, 1, 123, 0, 26, 1, 0, 0, 3533, 4, 0, 0, 3, 8190, 3, 0, 712, 2, 1, 3, 4, 1, 0, 8168, 1, 8187, 4940, 7064, 3, 2, 8189, 8186] payload_rows_per_dst=[16380, 16384, 15150, 16380, 16380, 16344, 14130, 16382] dup_rows_per_dst=[246, 44, 1804, 12, 1430, 2, 0, 4] combine_rows_per_src=[16380, 16384, 15150, 16380, 16380, 16344, 14130, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=893337600 dispatch_meta_bytes=458560 combine_payload_bytes=893337600 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=893796160 ingress_bytes=1069498368 link_bytes=1069498368 token_bytes=1787133760 weight_movement_bytes=176160768 total_nvlink_bytes=1963294528 weight_to_token_ratio=0.098572 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.988744430e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=773537792 dispatch_meta_bytes=454472 combine_payload_bytes=773537792 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=773992264 ingress_bytes=1654341632 link_bytes=1654341632 token_bytes=1547530056 weight_movement_bytes=880803840 total_nvlink_bytes=2428333896 weight_to_token_ratio=0.569168 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.263622583e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=899579904 dispatch_meta_bytes=456408 combine_payload_bytes=899579904 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=900036312 ingress_bytes=1251901440 link_bytes=1251901440 token_bytes=1799616216 weight_movement_bytes=352321536 total_nvlink_bytes=2151937752 weight_to_token_ratio=0.195776 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=20864 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.010125495e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=852590592 dispatch_meta_bytes=474056 combine_payload_bytes=852590592 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=853064648 ingress_bytes=1909555200 link_bytes=1909555200 token_bytes=1705655240 weight_movement_bytes=1056964608 total_nvlink_bytes=2762619848 weight_to_token_ratio=0.619682 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=22912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.069271203e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=821624832 dispatch_meta_bytes=458632 combine_payload_bytes=821624832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=822083464 ingress_bytes=1173946368 link_bytes=1173946368 token_bytes=1643708296 weight_movement_bytes=352321536 total_nvlink_bytes=1996029832 weight_to_token_ratio=0.214346 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.573609631e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=713654272 dispatch_meta_bytes=387712 combine_payload_bytes=713654272 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=714041984 ingress_bytes=1770618880 link_bytes=1770618880 token_bytes=1427696256 weight_movement_bytes=1056964608 total_nvlink_bytes=2484660864 weight_to_token_ratio=0.740329 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.914726635e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=910835712 dispatch_meta_bytes=458904 combine_payload_bytes=910835712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=911294616 ingress_bytes=1263157248 link_bytes=1263157248 token_bytes=1822130328 weight_movement_bytes=352321536 total_nvlink_bytes=2174451864 weight_to_token_ratio=0.193357 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.073153321e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=100214 payload_rows_remote=91940 dup_rows=30858 dup_rows_remote=29466 dedup_ratio=0.235428 router_tokens_per_expert=[5, 2, 1470, 22, 244, 5, 4446, 5, 7, 7780, 5, 2044, 1889, 5730, 1, 7161, 4633, 7383, 2497, 1, 6, 3756, 780, 57, 18, 3653, 2248, 4019, 2069, 109, 3261, 230] payload_rows_per_dst=[15284, 12246, 14460, 8978, 8274, 14964, 13804, 12204] dup_rows_per_dst=[2480, 3596, 3542, 1154, 1392, 7222, 6072, 5400] combine_rows_per_src=[15284, 12246, 14460, 8978, 8274, 14964, 13804, 12204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118822 payload_rows_remote=102438 dup_rows=12250 dup_rows_remote=12226 dedup_ratio=0.093460 router_tokens_per_expert=[8192, 8192, 0, 0, 8182, 0, 6091, 8192, 0, 2103, 11, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8187, 8190, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16366, 4208, 16382, 16370, 16364, 16364, 16384] dup_rows_per_dst=[24, 12176, 20, 0, 14, 16, 0, 0] combine_rows_per_src=[16384, 16366, 4208, 16382, 16370, 16364, 16364, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127960 payload_rows_remote=111578 dup_rows=3112 dup_rows_remote=3112 dedup_ratio=0.023743 router_tokens_per_expert=[8188, 8189, 5, 2, 8186, 0, 6685, 8159, 0, 1509, 35, 4, 0, 4, 0, 8187, 1, 4, 2, 0, 8182, 8179, 1, 9, 0, 5, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 13370, 16384, 16382, 16374, 16368, 16380, 16318] dup_rows_per_dst=[8, 0, 3084, 0, 6, 10, 4, 0] combine_rows_per_src=[16384, 13370, 16384, 16382, 16374, 16368, 16380, 16318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125108 payload_rows_remote=110456 dup_rows=5964 dup_rows_remote=1744 dedup_ratio=0.045502 router_tokens_per_expert=[8191, 8190, 0, 1, 8161, 0, 5119, 8190, 2, 3082, 19, 0, 0, 0, 0, 8190, 0, 1, 2, 0, 8191, 8191, 0, 0, 1, 1, 0, 0, 3, 1, 0, 0] payload_rows_per_dst=[16382, 12162, 14652, 16380, 16384, 16382, 16384, 16382] dup_rows_per_dst=[0, 1732, 4220, 0, 4, 0, 2, 6] combine_rows_per_src=[16382, 12162, 14652, 16380, 16384, 16382, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126404 payload_rows_remote=110026 dup_rows=4668 dup_rows_remote=4668 dedup_ratio=0.035614 router_tokens_per_expert=[8189, 8192, 2, 0, 8190, 0, 5902, 8156, 0, 2322, 1, 8, 0, 0, 0, 8189, 0, 0, 3, 0, 8187, 8185, 5, 4, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[16384, 11804, 16384, 16378, 16380, 16382, 16378, 16314] dup_rows_per_dst=[4, 0, 4658, 0, 0, 6, 0, 0] combine_rows_per_src=[16384, 11804, 16384, 16378, 16380, 16382, 16378, 16314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119008 payload_rows_remote=102950 dup_rows=12064 dup_rows_remote=332 dedup_ratio=0.092041 router_tokens_per_expert=[8192, 8191, 1, 0, 8191, 0, 6831, 8188, 0, 1364, 0, 0, 1, 0, 1, 8192, 0, 0, 0, 0, 8191, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16058, 4656, 16384, 16382, 16384, 16384, 16376] dup_rows_per_dst=[0, 11732, 326, 4, 0, 0, 0, 2] combine_rows_per_src=[16384, 16058, 4656, 16384, 16382, 16384, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127714 payload_rows_remote=111332 dup_rows=3358 dup_rows_remote=3098 dedup_ratio=0.025620 router_tokens_per_expert=[8189, 8184, 0, 2, 8188, 1, 6677, 8190, 0, 1463, 46, 2, 4, 3, 0, 8189, 2, 3, 0, 3, 8189, 8187, 2, 0, 3, 0, 2, 6, 0, 0, 1, 0] payload_rows_per_dst=[16352, 13072, 16380, 16384, 16382, 16378, 16384, 16382] dup_rows_per_dst=[0, 0, 3018, 26, 10, 26, 18, 260] combine_rows_per_src=[16352, 13072, 16380, 16384, 16382, 16378, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=839172096 dispatch_meta_bytes=458656 combine_payload_bytes=839172096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=839630752 ingress_bytes=1719975936 link_bytes=1719975936 token_bytes=1678802848 weight_movement_bytes=880803840 total_nvlink_bytes=2559606688 weight_to_token_ratio=0.524662 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.631147288e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=914046976 dispatch_meta_bytes=458760 combine_payload_bytes=914046976 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=914505736 ingress_bytes=1090207744 link_bytes=1090207744 token_bytes=1828552712 weight_movement_bytes=176160768 total_nvlink_bytes=2004713480 weight_to_token_ratio=0.096339 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.104708291e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128212 payload_rows_remote=111838 dup_rows=2860 dup_rows_remote=2858 dedup_ratio=0.021820 router_tokens_per_expert=[8188, 8185, 3, 2, 8190, 0, 6783, 8177, 2, 1385, 34, 3, 2, 1, 0, 8187, 2, 1, 1, 2, 8186, 8185, 0, 3, 2, 1, 0, 3, 0, 6, 0, 2] payload_rows_per_dst=[16376, 13566, 16384, 16380, 16378, 16374, 16384, 16370] dup_rows_per_dst=[4, 0, 2844, 0, 4, 2, 6, 0] combine_rows_per_src=[16376, 13566, 16384, 16380, 16378, 16374, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=904855552 dispatch_meta_bytes=448800 combine_payload_bytes=904855552 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=905304352 ingress_bytes=1257177088 link_bytes=1257177088 token_bytes=1810159904 weight_movement_bytes=352321536 total_nvlink_bytes=2162481440 weight_to_token_ratio=0.194636 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=25600 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.039666921e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 0, 1, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=901332992 dispatch_meta_bytes=458776 combine_payload_bytes=901332992 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=901791768 ingress_bytes=1253654528 link_bytes=1253654528 token_bytes=1803124760 weight_movement_bytes=352321536 total_nvlink_bytes=2155446296 weight_to_token_ratio=0.195395 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.019942056e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=753172480 dispatch_meta_bytes=485624 combine_payload_bytes=753172480 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=753658104 ingress_bytes=1810137088 link_bytes=1810137088 token_bytes=1506830584 weight_movement_bytes=1056964608 total_nvlink_bytes=2563795192 weight_to_token_ratio=0.701449 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=17024 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.013601210e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=843366400 dispatch_meta_bytes=413128 combine_payload_bytes=843366400 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=843779528 ingress_bytes=2604974080 link_bytes=2604974080 token_bytes=1687145928 weight_movement_bytes=1761607680 total_nvlink_bytes=3448753608 weight_to_token_ratio=1.044135 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.458676748e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=912031744 dispatch_meta_bytes=457720 combine_payload_bytes=912031744 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=912489464 ingress_bytes=1616674816 link_bytes=1616674816 token_bytes=1824521208 weight_movement_bytes=704643072 total_nvlink_bytes=2529164280 weight_to_token_ratio=0.386207 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.052704136e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=916176896 dispatch_meta_bytes=458784 combine_payload_bytes=916176896 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=916635680 ingress_bytes=1796980736 link_bytes=1796980736 token_bytes=1832812576 weight_movement_bytes=880803840 total_nvlink_bytes=2713616416 weight_to_token_ratio=0.480575 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.006234202e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130224 payload_rows_remote=113840 dup_rows=848 dup_rows_remote=824 dedup_ratio=0.006470 router_tokens_per_expert=[0, 8187, 8192, 0, 0, 0, 8192, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 58, 0, 0, 8191, 276, 0, 8147, 4, 4, 112, 0, 0, 0, 7788, 0, 8192] payload_rows_per_dst=[16384, 16382, 16372, 16362, 16382, 16382, 16384, 15576] dup_rows_per_dst=[24, 4, 0, 0, 116, 472, 232, 0] combine_rows_per_src=[16384, 16382, 16372, 16362, 16382, 16382, 16384, 15576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130116 payload_rows_remote=113734 dup_rows=956 dup_rows_remote=946 dedup_ratio=0.007294 router_tokens_per_expert=[5, 8184, 8187, 4, 1, 0, 8185, 13, 0, 2, 4, 8187, 0, 3, 4, 2, 2, 0, 1, 8182, 433, 0, 7996, 6, 78, 93, 4, 9, 0, 7775, 2, 8174] payload_rows_per_dst=[16374, 16378, 16384, 16382, 16366, 16294, 16384, 15554] dup_rows_per_dst=[12, 20, 2, 10, 4, 576, 332, 0] combine_rows_per_src=[16374, 16378, 16384, 16382, 16366, 16294, 16384, 15554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130032 payload_rows_remote=113648 dup_rows=1040 dup_rows_remote=1040 dedup_ratio=0.007935 router_tokens_per_expert=[1, 8191, 8191, 1, 1, 0, 8191, 0, 0, 0, 1, 8191, 0, 2, 0, 0, 88, 2, 0, 8191, 274, 0, 8097, 1, 1, 210, 0, 1, 0, 7710, 0, 8191] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16382, 16312, 16384, 15420] dup_rows_per_dst=[4, 0, 0, 2, 180, 432, 422, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16382, 16312, 16384, 15420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129780 payload_rows_remote=113410 dup_rows=1292 dup_rows_remote=1210 dedup_ratio=0.009857 router_tokens_per_expert=[0, 8192, 8192, 0, 0, 0, 8182, 14, 2, 0, 0, 8192, 0, 0, 0, 0, 20, 16, 0, 8189, 633, 2, 8104, 10, 34, 8, 0, 0, 3, 7559, 0, 8184] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16382, 16372, 16370, 15124] dup_rows_per_dst=[0, 12, 4, 0, 68, 1126, 82, 0] combine_rows_per_src=[16384, 16380, 16384, 16384, 16382, 16372, 16370, 15124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=932577280 dispatch_meta_bytes=458656 combine_payload_bytes=932577280 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=933035936 ingress_bytes=1813381120 link_bytes=1813381120 token_bytes=1865613216 weight_movement_bytes=880803840 total_nvlink_bytes=2746417056 weight_to_token_ratio=0.472126 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.015417733e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130356 payload_rows_remote=113972 dup_rows=716 dup_rows_remote=716 dedup_ratio=0.005463 router_tokens_per_expert=[0, 8190, 8192, 0, 0, 0, 8191, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 23, 2, 0, 8191, 297, 0, 8155, 2, 13, 44, 0, 0, 0, 7852, 1, 8190] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16350, 16384, 15706] dup_rows_per_dst=[0, 0, 0, 0, 48, 558, 110, 0] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16350, 16384, 15706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124200 payload_rows_remote=108056 dup_rows=6872 dup_rows_remote=298 dedup_ratio=0.052429 router_tokens_per_expert=[4, 8189, 8189, 3, 3, 2, 8187, 0, 2, 0, 3, 8189, 0, 0, 4, 2, 1, 9, 3, 8189, 96, 0, 8173, 0, 20, 32, 0, 3, 0, 8058, 0, 8175] payload_rows_per_dst=[16376, 16362, 16384, 16384, 15820, 12436, 14294, 16144] dup_rows_per_dst=[8, 4, 4, 6, 22, 174, 80, 6574] combine_rows_per_src=[16376, 16362, 16384, 16384, 15820, 12436, 14294, 16144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=931708928 dispatch_meta_bytes=458720 combine_payload_bytes=931708928 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=932167648 ingress_bytes=1284030464 link_bytes=1284030464 token_bytes=1863876576 weight_movement_bytes=352321536 total_nvlink_bytes=2216198112 weight_to_token_ratio=0.189026 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.190034618e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=931004416 dispatch_meta_bytes=458752 combine_payload_bytes=931004416 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=931463168 ingress_bytes=1283325952 link_bytes=1283325952 token_bytes=1862467584 weight_movement_bytes=352321536 total_nvlink_bytes=2214789120 weight_to_token_ratio=0.189169 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.186089645e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129994 payload_rows_remote=113642 dup_rows=1078 dup_rows_remote=638 dedup_ratio=0.008224 router_tokens_per_expert=[4, 8185, 8190, 1, 2, 0, 8186, 2, 0, 1, 2, 8190, 0, 1, 2, 2, 59, 1, 1, 8188, 220, 1, 8171, 4, 43, 213, 1, 0, 2, 7679, 0, 8185] payload_rows_per_dst=[16374, 16378, 16384, 16384, 16378, 16352, 16382, 15362] dup_rows_per_dst=[6, 2, 2, 6, 120, 440, 502, 0] combine_rows_per_src=[16374, 16378, 16384, 16384, 16378, 16352, 16382, 15362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=929054720 dispatch_meta_bytes=458480 combine_payload_bytes=929054720 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=929513200 ingress_bytes=1281376256 link_bytes=1281376256 token_bytes=1858567920 weight_movement_bytes=352321536 total_nvlink_bytes=2210889456 weight_to_token_ratio=0.189566 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.175172161e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123724 payload_rows_remote=107344 dup_rows=7348 dup_rows_remote=5564 dedup_ratio=0.056061 router_tokens_per_expert=[8192, 0, 125, 0, 1, 0, 0, 0, 3793, 24, 3, 0, 0, 8179, 0, 0, 892, 0, 0, 0, 0, 0, 0, 8002, 0, 8187, 6607, 5150, 0, 0, 8191, 8190] payload_rows_per_dst=[16384, 16374, 15542, 16358, 16380, 16004, 10300, 16382] dup_rows_per_dst=[250, 2, 5312, 0, 1784, 0, 0, 0] combine_rows_per_src=[16384, 16374, 15542, 16358, 16380, 16004, 10300, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=933658624 dispatch_meta_bytes=458752 combine_payload_bytes=933658624 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=934117376 ingress_bytes=1462140928 link_bytes=1462140928 token_bytes=1867776000 weight_movement_bytes=528482304 total_nvlink_bytes=2396258304 weight_to_token_ratio=0.282947 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.187378869e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=885194752 dispatch_meta_bytes=433416 combine_payload_bytes=885194752 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=885628168 ingress_bytes=1589837824 link_bytes=1589837824 token_bytes=1770822920 weight_movement_bytes=704643072 total_nvlink_bytes=2475465992 weight_to_token_ratio=0.397918 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.902428183e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=930955264 dispatch_meta_bytes=457120 combine_payload_bytes=930955264 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=931412384 ingress_bytes=1635598336 link_bytes=1635598336 token_bytes=1862367648 weight_movement_bytes=704643072 total_nvlink_bytes=2567010720 weight_to_token_ratio=0.378359 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.158667949e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122172 payload_rows_remote=105788 dup_rows=8900 dup_rows_remote=8866 dedup_ratio=0.067902 router_tokens_per_expert=[0, 0, 0, 6, 8192, 0, 7181, 0, 1011, 0, 8187, 0, 0, 0, 0, 1, 0, 8192, 8192, 0, 0, 8191, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15546, 8356, 16384, 16384, 16352, 16384, 16382] dup_rows_per_dst=[34, 8026, 830, 0, 10, 0, 0, 0] combine_rows_per_src=[16384, 15546, 8356, 16384, 16384, 16352, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=879362048 dispatch_meta_bytes=451632 combine_payload_bytes=879362048 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=879813680 ingress_bytes=1231683584 link_bytes=1231683584 token_bytes=1759175728 weight_movement_bytes=352321536 total_nvlink_bytes=2111497264 weight_to_token_ratio=0.200276 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=26880 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.896913940e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 6, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130424 payload_rows_remote=114072 dup_rows=648 dup_rows_remote=630 dedup_ratio=0.004944 router_tokens_per_expert=[4, 4, 5, 12, 8184, 0, 7915, 0, 292, 2, 8172, 12, 5, 8, 4, 1, 4, 8186, 8185, 0, 2, 8184, 0, 0, 8182, 8167, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[16378, 15830, 16370, 16352, 16376, 16372, 16364, 16382] dup_rows_per_dst=[40, 0, 586, 18, 2, 0, 0, 2] combine_rows_per_src=[16378, 15830, 16370, 16352, 16376, 16372, 16364, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130694 payload_rows_remote=114310 dup_rows=378 dup_rows_remote=20 dedup_ratio=0.002884 router_tokens_per_expert=[1, 1, 1, 0, 8188, 1, 8013, 0, 186, 0, 8184, 1, 1, 1, 1, 3, 3, 8190, 8191, 0, 1, 8191, 0, 0, 8187, 8190, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16026, 16384, 16384, 16382, 16384, 16376, 16380] dup_rows_per_dst=[4, 2, 358, 8, 6, 0, 0, 0] combine_rows_per_src=[16378, 16026, 16384, 16384, 16382, 16384, 16376, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=866615296 dispatch_meta_bytes=458616 combine_payload_bytes=866615296 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=867073912 ingress_bytes=1395097600 link_bytes=1395097600 token_bytes=1733689208 weight_movement_bytes=528482304 total_nvlink_bytes=2262171512 weight_to_token_ratio=0.304831 gemm_rows_defined=66688 gemm_rows_charged=67584 gemm_tail_pad_rows=896 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.013258 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.811964218e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128392 payload_rows_remote=112030 dup_rows=2680 dup_rows_remote=2680 dedup_ratio=0.020447 router_tokens_per_expert=[0, 0, 0, 12, 8154, 0, 6889, 0, 1323, 5, 8191, 0, 0, 0, 0, 1, 0, 8192, 8192, 0, 0, 8184, 15, 5, 8181, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16332, 13778, 16384, 16384, 16384, 16384, 16362, 16384] dup_rows_per_dst=[0, 0, 2654, 2, 0, 24, 0, 0] combine_rows_per_src=[16332, 13778, 16384, 16384, 16384, 16384, 16362, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129024 payload_rows_remote=114682 dup_rows=2048 dup_rows_remote=2048 dedup_ratio=0.015625 router_tokens_per_expert=[0, 0, 0, 0, 8191, 0, 7171, 0, 1021, 1, 8191, 0, 0, 0, 0, 2, 0, 8192, 8192, 0, 0, 8191, 1, 0, 8191, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 14342, 16382, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 0, 2044, 4, 0, 0, 0, 0] combine_rows_per_src=[16382, 14342, 16382, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130368 payload_rows_remote=113986 dup_rows=704 dup_rows_remote=704 dedup_ratio=0.005371 router_tokens_per_expert=[3, 3, 5, 0, 8186, 0, 7942, 1, 253, 0, 8186, 4, 3, 3, 2, 0, 2, 8187, 8188, 1, 3, 8189, 0, 0, 8189, 8179, 0, 4, 0, 1, 2, 0] payload_rows_per_dst=[16334, 15892, 16290, 16342, 16378, 16384, 16366, 16382] dup_rows_per_dst=[14, 40, 502, 10, 6, 6, 126, 0] combine_rows_per_src=[16334, 15892, 16290, 16342, 16378, 16384, 16366, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=934477824 dispatch_meta_bytes=458808 combine_payload_bytes=934477824 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934936632 ingress_bytes=1286799360 link_bytes=1286799360 token_bytes=1869414456 weight_movement_bytes=352321536 total_nvlink_bytes=2221735992 weight_to_token_ratio=0.188466 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.205539280e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=936427520 dispatch_meta_bytes=457320 combine_payload_bytes=936427520 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=936884840 ingress_bytes=2169552896 link_bytes=2169552896 token_bytes=1873312360 weight_movement_bytes=1233125376 total_nvlink_bytes=3106437736 weight_to_token_ratio=0.658259 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=30592 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.214859060e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130446 payload_rows_remote=114062 dup_rows=626 dup_rows_remote=622 dedup_ratio=0.004776 router_tokens_per_expert=[1, 1, 3, 9, 8190, 1, 7911, 0, 283, 0, 8186, 2, 2, 1, 2, 7, 0, 8188, 8189, 1, 2, 8185, 6, 1, 8181, 8178, 0, 0, 0, 1, 2, 3] payload_rows_per_dst=[16384, 15822, 16374, 16384, 16376, 16384, 16338, 16384] dup_rows_per_dst=[24, 2, 568, 20, 0, 4, 0, 8] combine_rows_per_src=[16384, 15822, 16374, 16384, 16376, 16384, 16338, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[3, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=917749760 dispatch_meta_bytes=458840 combine_payload_bytes=917749760 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=918208600 ingress_bytes=1622392832 link_bytes=1622392832 token_bytes=1835958360 weight_movement_bytes=704643072 total_nvlink_bytes=2540601432 weight_to_token_ratio=0.383801 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.084722639e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=466920 combine_payload_bytes=939474944 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939941864 ingress_bytes=1820278784 link_bytes=1820278784 token_bytes=1879416808 weight_movement_bytes=880803840 total_nvlink_bytes=2760220648 weight_to_token_ratio=0.468658 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019280137e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=933773312 dispatch_meta_bytes=458760 combine_payload_bytes=933773312 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934232072 ingress_bytes=1286094848 link_bytes=1286094848 token_bytes=1868005384 weight_movement_bytes=352321536 total_nvlink_bytes=2220326920 weight_to_token_ratio=0.188608 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.201594306e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=934395904 dispatch_meta_bytes=458736 combine_payload_bytes=934395904 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=934854640 ingress_bytes=1462878208 link_bytes=1462878208 token_bytes=1869250544 weight_movement_bytes=528482304 total_nvlink_bytes=2397732848 weight_to_token_ratio=0.282724 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.191507329e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130458 payload_rows_remote=114074 dup_rows=614 dup_rows_remote=34 dedup_ratio=0.004684 router_tokens_per_expert=[0, 8192, 8152, 8192, 0, 0, 0, 0, 0, 4, 8192, 0, 0, 8192, 36, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 15804, 16382, 16378, 16378, 16364] dup_rows_per_dst=[580, 0, 34, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 15804, 16382, 16378, 16378, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123600 payload_rows_remote=107216 dup_rows=7472 dup_rows_remote=7472 dedup_ratio=0.057007 router_tokens_per_expert=[8192, 8192, 1, 0, 8079, 0, 4517, 8139, 0, 3847, 1, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 9034, 16384, 16372, 16384, 16380, 16384, 16278] dup_rows_per_dst=[2, 0, 7470, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 9034, 16384, 16372, 16384, 16380, 16384, 16278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130842 payload_rows_remote=114532 dup_rows=230 dup_rows_remote=150 dedup_ratio=0.001755 router_tokens_per_expert=[25, 8163, 8146, 8188, 0, 6, 0, 4, 6, 32, 8183, 0, 0, 8155, 40, 0, 3, 0, 6, 4, 0, 8184, 1, 8186, 5, 5, 3, 0, 8183, 5, 0, 3] payload_rows_per_dst=[16296, 16384, 16384, 16310, 16338, 16370, 16384, 16376] dup_rows_per_dst=[46, 8, 58, 80, 14, 0, 18, 6] combine_rows_per_src=[16296, 16384, 16384, 16310, 16338, 16370, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130972 payload_rows_remote=114588 dup_rows=100 dup_rows_remote=98 dedup_ratio=0.000763 router_tokens_per_expert=[0, 8190, 8145, 8191, 0, 1, 0, 1, 1, 5, 8187, 0, 0, 8190, 45, 0, 0, 1, 1, 1, 1, 8191, 0, 8191, 0, 1, 0, 0, 8191, 1, 0, 1] payload_rows_per_dst=[16290, 16384, 16384, 16380, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 2, 2, 90, 2, 2, 0, 2] combine_rows_per_src=[16290, 16384, 16384, 16380, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=934494208 dispatch_meta_bytes=456432 combine_payload_bytes=934494208 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=934950640 ingress_bytes=2167619584 link_bytes=2167619584 token_bytes=1869444848 weight_movement_bytes=1233125376 total_nvlink_bytes=3102570224 weight_to_token_ratio=0.659621 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.213776486e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130958 payload_rows_remote=114582 dup_rows=114 dup_rows_remote=114 dedup_ratio=0.000870 router_tokens_per_expert=[0, 8192, 8139, 8188, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 53, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 4, 0] payload_rows_per_dst=[16278, 16384, 16384, 16384, 16384, 16384, 16376, 16384] dup_rows_per_dst=[0, 0, 0, 106, 0, 0, 0, 8] combine_rows_per_src=[16278, 16384, 16384, 16384, 16384, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=878313472 dispatch_meta_bytes=458752 combine_payload_bytes=878313472 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=878772224 ingress_bytes=1406795776 link_bytes=1406795776 token_bytes=1757085696 weight_movement_bytes=528482304 total_nvlink_bytes=2285568000 weight_to_token_ratio=0.300772 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.877469120e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130716 payload_rows_remote=114338 dup_rows=356 dup_rows_remote=348 dedup_ratio=0.002716 router_tokens_per_expert=[6, 8179, 8189, 8188, 0, 6, 0, 3, 4, 31, 8186, 0, 0, 8160, 0, 0, 0, 0, 5, 3, 0, 8187, 5, 8188, 0, 3, 0, 2, 8184, 5, 0, 2] payload_rows_per_dst=[16340, 16352, 16190, 16320, 16368, 16384, 16384, 16378] dup_rows_per_dst=[12, 10, 58, 194, 44, 28, 2, 8] combine_rows_per_src=[16340, 16352, 16190, 16320, 16368, 16384, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114654 dup_rows=34 dup_rows_remote=32 dedup_ratio=0.000259 router_tokens_per_expert=[0, 8192, 8176, 8192, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 16, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 8191, 0, 1, 0] payload_rows_per_dst=[16352, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 32, 0, 0, 0, 0] combine_rows_per_src=[16352, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130980 payload_rows_remote=114602 dup_rows=92 dup_rows_remote=92 dedup_ratio=0.000702 router_tokens_per_expert=[1, 8186, 8175, 8184, 0, 4, 1, 2, 2, 17, 8189, 0, 0, 8176, 10, 0, 2, 1, 3, 2, 0, 8189, 0, 8190, 0, 2, 2, 1, 8187, 2, 7, 1] payload_rows_per_dst=[16350, 16384, 16384, 16352, 16378, 16378, 16374, 16380] dup_rows_per_dst=[2, 10, 32, 20, 10, 0, 4, 14] combine_rows_per_src=[16350, 16384, 16384, 16352, 16378, 16378, 16374, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938246144 dispatch_meta_bytes=458728 combine_payload_bytes=938246144 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=938704872 ingress_bytes=1642889216 link_bytes=1642889216 token_bytes=1876951016 weight_movement_bytes=704643072 total_nvlink_bytes=2581594088 weight_to_token_ratio=0.375419 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.026515 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.199493834e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938704896 dispatch_meta_bytes=458744 combine_payload_bytes=938704896 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939163640 ingress_bytes=1643347968 link_bytes=1643347968 token_bytes=1877868536 weight_movement_bytes=704643072 total_nvlink_bytes=2582511608 weight_to_token_ratio=0.375236 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202062654e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938655744 dispatch_meta_bytes=458784 combine_payload_bytes=938655744 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939114528 ingress_bytes=1290977280 link_bytes=1290977280 token_bytes=1877770272 weight_movement_bytes=352321536 total_nvlink_bytes=2230091808 weight_to_token_ratio=0.187628 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228933888e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=936656896 dispatch_meta_bytes=458744 combine_payload_bytes=936656896 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=937115640 ingress_bytes=1465139200 link_bytes=1465139200 token_bytes=1873772536 weight_movement_bytes=528482304 total_nvlink_bytes=2402254840 weight_to_token_ratio=0.282042 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.204167941e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=458744 combine_payload_bytes=939245568 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939704312 ingress_bytes=1291567104 link_bytes=1291567104 token_bytes=1878949880 weight_movement_bytes=352321536 total_nvlink_bytes=2231271416 weight_to_token_ratio=0.187510 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232236656e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131002 payload_rows_remote=114618 dup_rows=70 dup_rows_remote=62 dedup_ratio=0.000534 router_tokens_per_expert=[0, 0, 0, 0, 7, 0, 0, 1, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16348, 16384, 16368, 16368] dup_rows_per_dst=[8, 16, 16, 18, 0, 12, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16382, 16348, 16384, 16368, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938819584 dispatch_meta_bytes=458776 combine_payload_bytes=938819584 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=939278360 ingress_bytes=1819623424 link_bytes=1819623424 token_bytes=1878097944 weight_movement_bytes=880803840 total_nvlink_bytes=2758901784 weight_to_token_ratio=0.468987 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018913163e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938950656 dispatch_meta_bytes=458720 combine_payload_bytes=938950656 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939409376 ingress_bytes=1467432960 link_bytes=1467432960 token_bytes=1878360032 weight_movement_bytes=528482304 total_nvlink_bytes=2406842336 weight_to_token_ratio=0.281353 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217012039e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130268 payload_rows_remote=113888 dup_rows=804 dup_rows_remote=764 dedup_ratio=0.006134 router_tokens_per_expert=[0, 8182, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 20, 0, 0, 8190, 348, 0, 8046, 4, 53, 120, 0, 0, 0, 7806, 0, 8192] payload_rows_per_dst=[16364, 16382, 16384, 16384, 16380, 16378, 16384, 15612] dup_rows_per_dst=[0, 0, 0, 0, 40, 418, 346, 0] combine_rows_per_src=[16364, 16382, 16384, 16384, 16380, 16378, 16384, 15612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114676 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000107 router_tokens_per_expert=[0, 0, 0, 0, 2, 2, 2, 1, 8190, 8191, 0, 1, 0, 0, 1, 8191, 0, 8190, 8190, 0, 1, 8191, 0, 0, 1, 0, 0, 8191, 0, 1, 1, 8189] payload_rows_per_dst=[16380, 16384, 16382, 16384, 16380, 16384, 16384, 16380] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[16380, 16384, 16382, 16384, 16380, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130996 payload_rows_remote=114630 dup_rows=76 dup_rows_remote=76 dedup_ratio=0.000580 router_tokens_per_expert=[0, 2, 0, 4, 3, 4, 3, 11, 8181, 8184, 0, 0, 0, 0, 5, 8178, 0, 8185, 8182, 0, 7, 8184, 6, 0, 7, 0, 8, 8184, 1, 5, 6, 8186] payload_rows_per_dst=[16376, 16382, 16362, 16366, 16364, 16378, 16384, 16384] dup_rows_per_dst=[6, 28, 0, 0, 0, 16, 14, 12] combine_rows_per_src=[16376, 16382, 16362, 16366, 16364, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131034 payload_rows_remote=114668 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000290 router_tokens_per_expert=[0, 0, 0, 0, 13, 0, 0, 0, 8192, 8191, 0, 0, 1, 6, 0, 8192, 0, 8190, 8192, 0, 0, 8184, 0, 0, 0, 4, 0, 8179, 0, 0, 0, 8192] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16368, 16366, 16384] dup_rows_per_dst=[0, 24, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16368, 16366, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130848 payload_rows_remote=114464 dup_rows=224 dup_rows_remote=212 dedup_ratio=0.001709 router_tokens_per_expert=[2, 5, 0, 0, 0, 5, 2, 8, 8187, 8186, 0, 0, 0, 0, 3, 8180, 1, 8187, 8185, 0, 5, 8187, 0, 3, 6, 0, 0, 8187, 1, 4, 4, 8188] payload_rows_per_dst=[16378, 16224, 16374, 16364, 16372, 16370, 16382, 16384] dup_rows_per_dst=[6, 20, 36, 0, 72, 6, 72, 12] combine_rows_per_src=[16378, 16224, 16374, 16364, 16372, 16370, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114678 dup_rows=10 dup_rows_remote=0 dedup_ratio=0.000076 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 3, 8190, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16382, 16380, 16384] dup_rows_per_dst=[0, 10, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16382, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130998 payload_rows_remote=114620 dup_rows=74 dup_rows_remote=64 dedup_ratio=0.000565 router_tokens_per_expert=[1, 1, 1, 0, 6, 2, 1, 14, 8185, 8190, 0, 0, 4, 3, 3, 8186, 0, 8188, 8189, 3, 3, 8187, 4, 0, 4, 0, 0, 8172, 2, 0, 2, 8185] payload_rows_per_dst=[16384, 16384, 16370, 16378, 16378, 16378, 16352, 16374] dup_rows_per_dst=[2, 38, 0, 14, 6, 10, 0, 4] combine_rows_per_src=[16384, 16384, 16370, 16378, 16378, 16378, 16352, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=932970496 dispatch_meta_bytes=458608 combine_payload_bytes=932970496 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=933429104 ingress_bytes=1637613568 link_bytes=1637613568 token_bytes=1866399600 weight_movement_bytes=704643072 total_nvlink_bytes=2571042672 weight_to_token_ratio=0.377541 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.169952407e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458760 combine_payload_bytes=939425792 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939884552 ingress_bytes=1996390400 link_bytes=1996390400 token_bytes=1879310344 weight_movement_bytes=1056964608 total_nvlink_bytes=2936274952 weight_to_token_ratio=0.562422 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117895291e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458824 combine_payload_bytes=939360256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939819080 ingress_bytes=1291681792 link_bytes=1291681792 token_bytes=1879179336 weight_movement_bytes=352321536 total_nvlink_bytes=2231500872 weight_to_token_ratio=0.187487 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232878861e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939048960 dispatch_meta_bytes=458824 combine_payload_bytes=939048960 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939507784 ingress_bytes=1467531264 link_bytes=1467531264 token_bytes=1878556744 weight_movement_bytes=528482304 total_nvlink_bytes=2407039048 weight_to_token_ratio=0.281324 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217562501e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130246 payload_rows_remote=113862 dup_rows=826 dup_rows_remote=826 dedup_ratio=0.006302 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0, 1, 8186, 0, 0, 0, 0, 8184, 8192, 8192, 0, 0, 0, 8, 8192, 5, 0, 0, 8186, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16370, 15594, 16384, 16366] dup_rows_per_dst=[0, 24, 648, 144, 10, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16380, 16370, 15594, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=937689088 dispatch_meta_bytes=458704 combine_payload_bytes=937689088 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=938147792 ingress_bytes=1642332160 link_bytes=1642332160 token_bytes=1875836880 weight_movement_bytes=704643072 total_nvlink_bytes=2580479952 weight_to_token_ratio=0.375642 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196374553e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458712 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939900888 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879343064 weight_movement_bytes=352321536 total_nvlink_bytes=2231664600 weight_to_token_ratio=0.187471 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938967040 dispatch_meta_bytes=458736 combine_payload_bytes=938967040 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=939425776 ingress_bytes=1467449344 link_bytes=1467449344 token_bytes=1878392816 weight_movement_bytes=528482304 total_nvlink_bytes=2406875120 weight_to_token_ratio=0.281348 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217103783e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=932757504 dispatch_meta_bytes=458752 combine_payload_bytes=932757504 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933216256 ingress_bytes=1285079040 link_bytes=1285079040 token_bytes=1865973760 weight_movement_bytes=352321536 total_nvlink_bytes=2218295296 weight_to_token_ratio=0.188814 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.195906206e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131010 payload_rows_remote=114676 dup_rows=62 dup_rows_remote=60 dedup_ratio=0.000473 router_tokens_per_expert=[0, 1, 0, 1, 8191, 8191, 0, 1, 1, 8166, 1, 0, 1, 0, 8188, 8190, 8191, 0, 1, 0, 0, 8191, 28, 1, 0, 8189, 0, 1, 1, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16334, 16378, 16384, 16384, 16380, 16382] dup_rows_per_dst=[2, 0, 2, 0, 0, 56, 0, 2] combine_rows_per_src=[16384, 16384, 16334, 16378, 16384, 16384, 16380, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130982 payload_rows_remote=114628 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.000687 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8176, 0, 0, 0, 0, 8178, 8192, 8192, 0, 0, 3, 12, 8190, 26, 0, 0, 8177, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[16384, 16384, 16352, 16356, 16384, 16384, 16354, 16384] dup_rows_per_dst=[0, 0, 0, 0, 6, 72, 0, 12] combine_rows_per_src=[16384, 16384, 16352, 16356, 16384, 16384, 16354, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130562 payload_rows_remote=114204 dup_rows=510 dup_rows_remote=510 dedup_ratio=0.003891 router_tokens_per_expert=[1, 7, 7, 2, 8186, 8179, 1, 6, 10, 7952, 6, 0, 0, 0, 8179, 8182, 8184, 0, 6, 0, 7, 8185, 225, 7, 0, 8176, 9, 10, 2, 0, 1, 6] payload_rows_per_dst=[16384, 16370, 15932, 16358, 16380, 16384, 16378, 16376] dup_rows_per_dst=[22, 2, 4, 0, 0, 464, 12, 6] combine_rows_per_src=[16384, 16370, 15932, 16358, 16380, 16384, 16378, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130432 payload_rows_remote=114048 dup_rows=640 dup_rows_remote=640 dedup_ratio=0.004883 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 7883, 0, 309, 0, 8186, 0, 0, 0, 0, 6, 0, 8192, 8192, 0, 0, 8192, 5, 0, 8187, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15766, 16372, 16384, 16384, 16384, 16374, 16384] dup_rows_per_dst=[0, 0, 618, 12, 0, 10, 0, 0] combine_rows_per_src=[16384, 15766, 16372, 16384, 16384, 16384, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130836 payload_rows_remote=114454 dup_rows=236 dup_rows_remote=236 dedup_ratio=0.001801 router_tokens_per_expert=[5, 4, 4, 0, 8187, 8184, 1, 5, 0, 8103, 6, 0, 0, 0, 8187, 8184, 8186, 0, 5, 0, 0, 8187, 86, 5, 0, 8186, 0, 6, 0, 1, 0, 4] payload_rows_per_dst=[16344, 16374, 16218, 16374, 16382, 16378, 16384, 16382] dup_rows_per_dst=[12, 2, 0, 14, 0, 172, 36, 0] combine_rows_per_src=[16344, 16374, 16218, 16374, 16382, 16378, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114672 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8187, 0, 0, 0, 0, 8191, 8192, 8192, 0, 0, 1, 2, 8192, 4, 0, 0, 8190, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16374, 16382, 16384, 16384, 16380, 16384] dup_rows_per_dst=[0, 0, 0, 0, 2, 12, 2, 0] combine_rows_per_src=[16384, 16384, 16374, 16382, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114654 dup_rows=34 dup_rows_remote=22 dedup_ratio=0.000259 router_tokens_per_expert=[0, 3, 3, 1, 8190, 8188, 0, 2, 0, 8184, 3, 0, 0, 0, 8185, 8187, 8189, 0, 2, 2, 3, 8188, 4, 3, 0, 8187, 1, 4, 1, 0, 4, 2] payload_rows_per_dst=[16382, 16380, 16374, 16370, 16382, 16384, 16382, 16384] dup_rows_per_dst=[6, 0, 0, 0, 4, 12, 2, 10] combine_rows_per_src=[16382, 16380, 16374, 16370, 16382, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458944 combine_payload_bytes=939425792 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939884736 ingress_bytes=1291747328 link_bytes=1291747328 token_bytes=1879310528 weight_movement_bytes=352321536 total_nvlink_bytes=2231632064 weight_to_token_ratio=0.187474 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233245835e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=458872 combine_payload_bytes=939032576 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939491448 ingress_bytes=1291354112 link_bytes=1291354112 token_bytes=1878524024 weight_movement_bytes=352321536 total_nvlink_bytes=2230845560 weight_to_token_ratio=0.187552 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231043989e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=935559168 dispatch_meta_bytes=458856 combine_payload_bytes=935559168 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=936018024 ingress_bytes=1640202240 link_bytes=1640202240 token_bytes=1871577192 weight_movement_bytes=704643072 total_nvlink_bytes=2576220264 weight_to_token_ratio=0.376497 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.184447890e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126774 payload_rows_remote=110390 dup_rows=4298 dup_rows_remote=4108 dedup_ratio=0.032791 router_tokens_per_expert=[8192, 0, 8192, 0, 8173, 0, 0, 0, 44, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8163, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16356, 12418, 16332, 16362, 16266, 16326, 16330] dup_rows_per_dst=[190, 4050, 56, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16356, 12418, 16332, 16362, 16266, 16326, 16330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937607168 dispatch_meta_bytes=458760 combine_payload_bytes=937607168 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938065928 ingress_bytes=1289928704 link_bytes=1289928704 token_bytes=1875673096 weight_movement_bytes=352321536 total_nvlink_bytes=2227994632 weight_to_token_ratio=0.187837 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223062300e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 2, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=458752 combine_payload_bytes=939393024 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939851776 ingress_bytes=1820196864 link_bytes=1820196864 token_bytes=1879244800 weight_movement_bytes=880803840 total_nvlink_bytes=2760048640 weight_to_token_ratio=0.468701 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019234265e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=934281216 dispatch_meta_bytes=458752 combine_payload_bytes=934281216 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=934739968 ingress_bytes=1638924288 link_bytes=1638924288 token_bytes=1869021184 weight_movement_bytes=704643072 total_nvlink_bytes=2573664256 weight_to_token_ratio=0.377012 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.177291892e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=458704 combine_payload_bytes=939245568 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=939704272 ingress_bytes=2348531712 link_bytes=2348531712 token_bytes=1878949840 weight_movement_bytes=1409286144 total_nvlink_bytes=3288235984 weight_to_token_ratio=0.750039 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315079726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=904314880 dispatch_meta_bytes=457992 combine_payload_bytes=904314880 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=904772872 ingress_bytes=1785118720 link_bytes=1785118720 token_bytes=1809087752 weight_movement_bytes=880803840 total_nvlink_bytes=2689891592 weight_to_token_ratio=0.486877 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.995919687e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130776 payload_rows_remote=114396 dup_rows=296 dup_rows_remote=18 dedup_ratio=0.002258 router_tokens_per_expert=[8191, 0, 8190, 0, 8164, 0, 1, 1, 143, 8184, 8190, 2, 1, 1, 0, 2, 0, 0, 0, 3, 2, 0, 2, 0, 1, 8078, 0, 0, 8191, 8189, 0, 0] payload_rows_per_dst=[16380, 16330, 16380, 16384, 16384, 16384, 16156, 16378] dup_rows_per_dst=[0, 2, 278, 4, 4, 6, 2, 0] combine_rows_per_src=[16380, 16330, 16380, 16384, 16384, 16384, 16156, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130742 payload_rows_remote=114604 dup_rows=330 dup_rows_remote=330 dedup_ratio=0.002518 router_tokens_per_expert=[8192, 0, 8192, 0, 8150, 0, 0, 0, 150, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8069, 0, 0, 8192, 8192, 0, 9] payload_rows_per_dst=[16384, 16300, 16384, 16384, 16384, 16384, 16138, 16384] dup_rows_per_dst=[0, 0, 300, 0, 0, 12, 0, 18] combine_rows_per_src=[16384, 16300, 16384, 16384, 16384, 16384, 16138, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129492 payload_rows_remote=113108 dup_rows=1580 dup_rows_remote=1556 dedup_ratio=0.012054 router_tokens_per_expert=[8183, 0, 8169, 0, 7479, 0, 6, 6, 746, 8173, 8183, 7, 6, 9, 0, 6, 0, 14, 0, 13, 20, 0, 9, 0, 0, 8146, 0, 0, 8181, 8180, 0, 0] payload_rows_per_dst=[16338, 14970, 16384, 16384, 16380, 16384, 16292, 16360] dup_rows_per_dst=[0, 12, 1468, 24, 36, 40, 0, 0] combine_rows_per_src=[16338, 14970, 16384, 16384, 16380, 16384, 16292, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127988 payload_rows_remote=111626 dup_rows=3084 dup_rows_remote=3004 dedup_ratio=0.023529 router_tokens_per_expert=[8186, 0, 8176, 0, 7454, 0, 5, 6, 775, 8186, 8183, 5, 6, 6, 0, 3, 0, 7, 0, 8, 25, 0, 6, 0, 0, 8135, 0, 1, 8182, 8181, 0, 0] payload_rows_per_dst=[16252, 14920, 15114, 16382, 16342, 16340, 16276, 16362] dup_rows_per_dst=[0, 10, 1542, 18, 20, 46, 1368, 80] combine_rows_per_src=[16252, 14920, 15114, 16382, 16342, 16340, 16276, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130558 payload_rows_remote=114622 dup_rows=514 dup_rows_remote=514 dedup_ratio=0.003922 router_tokens_per_expert=[8192, 0, 8192, 0, 7968, 0, 0, 0, 256, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 8192, 8192, 0, 1] payload_rows_per_dst=[16384, 15936, 16384, 16384, 16384, 16384, 16318, 16384] dup_rows_per_dst=[0, 0, 512, 0, 0, 0, 0, 2] combine_rows_per_src=[16384, 15936, 16384, 16384, 16384, 16384, 16318, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937132032 dispatch_meta_bytes=457656 combine_payload_bytes=937132032 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=937589688 ingress_bytes=2346418176 link_bytes=2346418176 token_bytes=1874721720 weight_movement_bytes=1409286144 total_nvlink_bytes=3284007864 weight_to_token_ratio=0.751731 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.313896234e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938835968 dispatch_meta_bytes=459736 combine_payload_bytes=938835968 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939295704 ingress_bytes=1291157504 link_bytes=1291157504 token_bytes=1878131672 weight_movement_bytes=352321536 total_nvlink_bytes=2230453208 weight_to_token_ratio=0.187591 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229943067e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=926580736 dispatch_meta_bytes=458656 combine_payload_bytes=926580736 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=927039392 ingress_bytes=1278902272 link_bytes=1278902272 token_bytes=1853620128 weight_movement_bytes=352321536 total_nvlink_bytes=2205941664 weight_to_token_ratio=0.190072 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.161318883e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130270 payload_rows_remote=113888 dup_rows=802 dup_rows_remote=778 dedup_ratio=0.006119 router_tokens_per_expert=[8189, 0, 8187, 1, 7915, 0, 2, 3, 372, 8189, 8188, 2, 3, 2, 0, 2, 0, 3, 0, 4, 15, 0, 3, 0, 0, 8075, 0, 1, 8185, 8187, 1, 7] payload_rows_per_dst=[16376, 15836, 16382, 16384, 16382, 16382, 16152, 16376] dup_rows_per_dst=[0, 4, 742, 8, 10, 24, 0, 14] combine_rows_per_src=[16376, 15836, 16382, 16384, 16382, 16382, 16152, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120664 payload_rows_remote=104280 dup_rows=10408 dup_rows_remote=10056 dedup_ratio=0.079407 router_tokens_per_expert=[8192, 5, 4, 0, 8192, 0, 8192, 0, 7892, 8189, 0, 0, 8192, 0, 0, 281, 0, 0, 0, 0, 0, 13, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16330, 6388, 16372, 16278, 16156, 16372] dup_rows_per_dst=[352, 34, 9958, 54, 0, 10, 0, 0] combine_rows_per_src=[16384, 16384, 16330, 6388, 16372, 16278, 16156, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130830 payload_rows_remote=114446 dup_rows=242 dup_rows_remote=242 dedup_ratio=0.001846 router_tokens_per_expert=[0, 8192, 8071, 8192, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 8191, 117, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16142, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 10, 232, 0, 0, 0, 0] combine_rows_per_src=[16142, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=914440192 dispatch_meta_bytes=458520 combine_payload_bytes=914440192 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=914898712 ingress_bytes=1795244032 link_bytes=1795244032 token_bytes=1829338904 weight_movement_bytes=880803840 total_nvlink_bytes=2710142744 weight_to_token_ratio=0.481488 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.005261721e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=460544 combine_payload_bytes=938983424 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939443968 ingress_bytes=1291304960 link_bytes=1291304960 token_bytes=1878427392 weight_movement_bytes=352321536 total_nvlink_bytes=2230748928 weight_to_token_ratio=0.187562 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230768759e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=932970496 dispatch_meta_bytes=458664 combine_payload_bytes=932970496 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933429160 ingress_bytes=1285292032 link_bytes=1285292032 token_bytes=1866399656 weight_movement_bytes=352321536 total_nvlink_bytes=2218721192 weight_to_token_ratio=0.188771 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.197098872e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=854261760 dispatch_meta_bytes=457344 combine_payload_bytes=854261760 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=854719104 ingress_bytes=1558904832 link_bytes=1558904832 token_bytes=1708980864 weight_movement_bytes=704643072 total_nvlink_bytes=2413623936 weight_to_token_ratio=0.412318 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.729216340e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=937541632 dispatch_meta_bytes=458752 combine_payload_bytes=937541632 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=938000384 ingress_bytes=1466023936 link_bytes=1466023936 token_bytes=1875542016 weight_movement_bytes=528482304 total_nvlink_bytes=2404024320 weight_to_token_ratio=0.281776 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.209122093e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130568 payload_rows_remote=114184 dup_rows=504 dup_rows_remote=500 dedup_ratio=0.003845 router_tokens_per_expert=[8189, 9, 0, 0, 8192, 0, 8192, 0, 7956, 8176, 0, 0, 8191, 0, 0, 235, 0, 0, 0, 0, 0, 10, 8192, 0, 0, 0, 8192, 2, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15912, 16384, 16384, 16384, 16384, 16352] dup_rows_per_dst=[12, 0, 0, 468, 0, 20, 4, 0] combine_rows_per_src=[16384, 16384, 15912, 16384, 16384, 16384, 16384, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129338 payload_rows_remote=114660 dup_rows=1734 dup_rows_remote=1734 dedup_ratio=0.013229 router_tokens_per_expert=[8191, 101, 62, 1, 8191, 1, 8191, 0, 7338, 8190, 0, 1, 8145, 1, 1, 738, 0, 0, 1, 0, 0, 0, 8190, 2, 0, 0, 8189, 0, 0, 0, 2, 0] payload_rows_per_dst=[16384, 16384, 14678, 16362, 16384, 16384, 16378, 16384] dup_rows_per_dst=[326, 0, 0, 1408, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 14678, 16362, 16384, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128504 payload_rows_remote=112124 dup_rows=2568 dup_rows_remote=232 dedup_ratio=0.019592 router_tokens_per_expert=[8184, 40, 20, 9, 8182, 1, 8183, 0, 6921, 8180, 0, 8, 8125, 4, 7, 1222, 6, 0, 6, 0, 17, 21, 8167, 6, 7, 0, 8183, 19, 3, 0, 15, 0] payload_rows_per_dst=[16384, 16366, 13858, 16380, 16372, 16380, 16380, 16384] dup_rows_per_dst=[122, 0, 0, 2336, 6, 42, 38, 24] combine_rows_per_src=[16384, 16366, 13858, 16380, 16372, 16380, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127024 payload_rows_remote=110642 dup_rows=4048 dup_rows_remote=3960 dedup_ratio=0.030884 router_tokens_per_expert=[8185, 129, 61, 6, 8186, 0, 8186, 1, 7048, 8187, 0, 6, 8106, 4, 5, 998, 5, 1, 4, 0, 12, 0, 8178, 4, 6, 0, 8185, 24, 1, 0, 8, 0] payload_rows_per_dst=[14778, 16374, 14028, 16372, 16384, 16324, 16382, 16382] dup_rows_per_dst=[380, 0, 0, 3458, 10, 6, 106, 88] combine_rows_per_src=[14778, 16374, 14028, 16372, 16384, 16324, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=465576 combine_payload_bytes=939294720 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939760296 ingress_bytes=1820098560 link_bytes=1820098560 token_bytes=1879055016 weight_movement_bytes=880803840 total_nvlink_bytes=2759858856 weight_to_token_ratio=0.468748 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019179219e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130674 payload_rows_remote=114290 dup_rows=398 dup_rows_remote=398 dedup_ratio=0.003036 router_tokens_per_expert=[8191, 2, 3, 0, 8192, 0, 8192, 0, 7995, 8190, 0, 0, 8192, 0, 0, 154, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 8192, 38, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15990, 16384, 16382, 16384, 16384, 16382] dup_rows_per_dst=[8, 0, 0, 308, 0, 6, 76, 0] combine_rows_per_src=[16384, 16384, 15990, 16384, 16382, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=935395328 dispatch_meta_bytes=458736 combine_payload_bytes=935395328 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=935854064 ingress_bytes=1640038400 link_bytes=1640038400 token_bytes=1871249392 weight_movement_bytes=704643072 total_nvlink_bytes=2575892464 weight_to_token_ratio=0.376563 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.183530454e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=918519808 dispatch_meta_bytes=449424 combine_payload_bytes=918519808 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=918969232 ingress_bytes=1623162880 link_bytes=1623162880 token_bytes=1837489040 weight_movement_bytes=704643072 total_nvlink_bytes=2542132112 weight_to_token_ratio=0.383482 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.089034586e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129050 payload_rows_remote=112666 dup_rows=2022 dup_rows_remote=2000 dedup_ratio=0.015427 router_tokens_per_expert=[8181, 119, 2, 3, 8189, 0, 8189, 0, 7195, 8183, 0, 3, 8157, 2, 3, 897, 2, 1, 2, 2, 9, 5, 8187, 2, 3, 1, 8186, 9, 0, 0, 4, 0] payload_rows_per_dst=[16384, 16378, 14396, 16378, 16384, 16384, 16378, 16368] dup_rows_per_dst=[226, 0, 0, 1740, 10, 22, 20, 4] combine_rows_per_src=[16384, 16378, 14396, 16378, 16384, 16384, 16378, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130580 payload_rows_remote=114196 dup_rows=492 dup_rows_remote=458 dedup_ratio=0.003754 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 105, 0, 0, 8192, 8190, 2, 0, 8087, 8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16376, 16384, 16016, 16286, 16370] dup_rows_per_dst=[34, 162, 8, 2, 286, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16376, 16384, 16016, 16286, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=906379264 dispatch_meta_bytes=458408 combine_payload_bytes=906379264 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=906837672 ingress_bytes=1434861568 link_bytes=1434861568 token_bytes=1813216936 weight_movement_bytes=528482304 total_nvlink_bytes=2341699240 weight_to_token_ratio=0.291461 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.034625840e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114674 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000107 router_tokens_per_expert=[0, 0, 0, 0, 6, 0, 0, 1, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16370, 16384] dup_rows_per_dst=[0, 14, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=936263680 dispatch_meta_bytes=458752 combine_payload_bytes=936263680 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=936722432 ingress_bytes=1640906752 link_bytes=1640906752 token_bytes=1872986112 weight_movement_bytes=704643072 total_nvlink_bytes=2577629184 weight_to_token_ratio=0.376214 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.188392863e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=922959872 dispatch_meta_bytes=458664 combine_payload_bytes=922959872 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=923418536 ingress_bytes=1627602944 link_bytes=1627602944 token_bytes=1846378408 weight_movement_bytes=704643072 total_nvlink_bytes=2551021480 weight_to_token_ratio=0.381635 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.113897091e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=935493632 dispatch_meta_bytes=458616 combine_payload_bytes=935493632 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=935952248 ingress_bytes=1640136704 link_bytes=1640136704 token_bytes=1871445880 weight_movement_bytes=704643072 total_nvlink_bytes=2576088952 weight_to_token_ratio=0.376523 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.184080916e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130622 payload_rows_remote=114240 dup_rows=450 dup_rows_remote=450 dedup_ratio=0.003433 router_tokens_per_expert=[2, 1, 8190, 0, 1, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 1, 8190, 213, 1, 1, 8191, 8182, 4, 1, 7974, 8191, 2, 7, 0, 8190, 0, 1, 2] payload_rows_per_dst=[16384, 16366, 16382, 16382, 16384, 15958, 16382, 16384] dup_rows_per_dst=[2, 0, 0, 0, 428, 0, 18, 2] combine_rows_per_src=[16384, 16366, 16382, 16382, 16384, 15958, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=458752 combine_payload_bytes=939409408 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=939868160 ingress_bytes=2172534784 link_bytes=2172534784 token_bytes=1879277568 weight_movement_bytes=1233125376 total_nvlink_bytes=3112402944 weight_to_token_ratio=0.656170 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216528793e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130514 payload_rows_remote=114130 dup_rows=558 dup_rows_remote=558 dedup_ratio=0.004257 router_tokens_per_expert=[0, 0, 8192, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 278, 0, 0, 8192, 8192, 0, 0, 7913, 8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 15826, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 0, 556, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 15826, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130756 payload_rows_remote=114372 dup_rows=316 dup_rows_remote=310 dedup_ratio=0.002411 router_tokens_per_expert=[11, 6, 8180, 4, 4, 0, 0, 0, 8181, 0, 1, 8, 0, 4, 9, 8182, 118, 9, 5, 8183, 8159, 1, 6, 8065, 8185, 17, 0, 0, 8183, 0, 0, 15] payload_rows_per_dst=[16382, 16326, 16378, 16384, 16384, 16144, 16374, 16384] dup_rows_per_dst=[20, 0, 2, 6, 246, 0, 30, 12] combine_rows_per_src=[16382, 16326, 16378, 16384, 16384, 16144, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=935854080 dispatch_meta_bytes=458760 combine_payload_bytes=935854080 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=936312840 ingress_bytes=1288175616 link_bytes=1288175616 token_bytes=1872166920 weight_movement_bytes=352321536 total_nvlink_bytes=2224488456 weight_to_token_ratio=0.188189 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.213245739e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127618 payload_rows_remote=111238 dup_rows=3454 dup_rows_remote=3450 dedup_ratio=0.026352 router_tokens_per_expert=[8, 3, 8185, 0, 3, 1, 0, 0, 8184, 2, 2, 6, 0, 1, 5, 8186, 111, 7, 4, 8186, 8164, 0, 3, 8080, 8186, 7, 10, 0, 8184, 0, 2, 6] payload_rows_per_dst=[16314, 16334, 16374, 16382, 13292, 16166, 16376, 16380] dup_rows_per_dst=[8, 2, 8, 2, 232, 3130, 68, 4] combine_rows_per_src=[16314, 16334, 16374, 16382, 13292, 16166, 16376, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=934952960 dispatch_meta_bytes=458752 combine_payload_bytes=934952960 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=935411712 ingress_bytes=1639596032 link_bytes=1639596032 token_bytes=1870364672 weight_movement_bytes=704643072 total_nvlink_bytes=2575007744 weight_to_token_ratio=0.376741 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.181053378e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936935424 dispatch_meta_bytes=458728 combine_payload_bytes=936935424 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=937394152 ingress_bytes=1289256960 link_bytes=1289256960 token_bytes=1874329576 weight_movement_bytes=352321536 total_nvlink_bytes=2226651112 weight_to_token_ratio=0.187972 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.219300814e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130678 payload_rows_remote=114294 dup_rows=394 dup_rows_remote=394 dedup_ratio=0.003006 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 197, 0, 0, 8192, 8192, 0, 0, 7995, 8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 15990, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 394, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 15990, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124088 payload_rows_remote=107704 dup_rows=6984 dup_rows_remote=6804 dedup_ratio=0.053284 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 8166, 5284, 8192, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 2928, 0, 0, 3] payload_rows_per_dst=[16384, 15442, 16374, 16384, 16226, 16370, 16384, 10524] dup_rows_per_dst=[180, 5854, 2, 6, 0, 0, 0, 942] combine_rows_per_src=[16384, 15442, 16374, 16384, 16226, 16370, 16384, 10524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130606 payload_rows_remote=114644 dup_rows=466 dup_rows_remote=462 dedup_ratio=0.003555 router_tokens_per_expert=[4, 2, 8188, 0, 2, 1, 0, 0, 8188, 0, 1, 3, 0, 1, 3, 8189, 219, 4, 2, 8189, 8172, 3, 1, 7979, 8188, 3, 2, 0, 8188, 0, 0, 4] payload_rows_per_dst=[16384, 16348, 16382, 16384, 16384, 15962, 16380, 16382] dup_rows_per_dst=[4, 2, 2, 2, 444, 4, 6, 2] combine_rows_per_src=[16384, 16348, 16382, 16384, 16384, 15962, 16380, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=911261696 dispatch_meta_bytes=458752 combine_payload_bytes=911261696 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=911720448 ingress_bytes=1263583232 link_bytes=1263583232 token_bytes=1822982144 weight_movement_bytes=352321536 total_nvlink_bytes=2175303680 weight_to_token_ratio=0.193267 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.075538654e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130996 payload_rows_remote=114612 dup_rows=76 dup_rows_remote=76 dedup_ratio=0.000580 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 0, 0, 34, 8191, 0, 0, 0, 0, 8158, 8192, 8192, 0, 0, 0, 3, 8192, 1, 0, 0, 8165, 24, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16316, 16384, 16384, 16378, 16384] dup_rows_per_dst=[0, 0, 68, 0, 0, 8, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16316, 16384, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936296448 dispatch_meta_bytes=458752 combine_payload_bytes=936296448 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=936755200 ingress_bytes=1640939520 link_bytes=1640939520 token_bytes=1873051648 weight_movement_bytes=704643072 total_nvlink_bytes=2577694720 weight_to_token_ratio=0.376201 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.188576350e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130216 payload_rows_remote=113834 dup_rows=856 dup_rows_remote=856 dedup_ratio=0.006531 router_tokens_per_expert=[8191, 8190, 0, 0, 0, 8183, 7786, 8172, 0, 1, 0, 8190, 2, 2, 0, 1, 0, 0, 1, 2, 0, 1, 8189, 1, 8191, 1, 0, 0, 411, 1, 0, 20] payload_rows_per_dst=[16380, 15572, 16382, 16350, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 4, 4, 2, 0, 846] combine_rows_per_src=[16380, 15572, 16382, 16350, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=882311168 dispatch_meta_bytes=458032 combine_payload_bytes=882311168 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=882769200 ingress_bytes=1939275776 link_bytes=1939275776 token_bytes=1765080368 weight_movement_bytes=1056964608 total_nvlink_bytes=2822044976 weight_to_token_ratio=0.598820 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.085913485e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=460424 combine_payload_bytes=939163648 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939624072 ingress_bytes=1996128256 link_bytes=1996128256 token_bytes=1878787720 weight_movement_bytes=1056964608 total_nvlink_bytes=2935752328 weight_to_token_ratio=0.562578 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117748501e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130204 payload_rows_remote=113820 dup_rows=868 dup_rows_remote=868 dedup_ratio=0.006622 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 8185, 7758, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 440, 0, 0, 1] payload_rows_per_dst=[16384, 15516, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 868] combine_rows_per_src=[16384, 15516, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130492 payload_rows_remote=114156 dup_rows=580 dup_rows_remote=570 dedup_ratio=0.004425 router_tokens_per_expert=[8184, 8180, 1, 3, 0, 7818, 7949, 8153, 6, 6, 0, 8181, 0, 12, 0, 8, 1, 0, 5, 38, 2, 8, 8182, 7, 8177, 7, 5, 1, 300, 0, 0, 302] payload_rows_per_dst=[16368, 15898, 16382, 16336, 16384, 16380, 16376, 16368] dup_rows_per_dst=[0, 0, 4, 10, 72, 18, 4, 472] combine_rows_per_src=[16368, 15898, 16382, 16336, 16384, 16380, 16376, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938901504 dispatch_meta_bytes=458752 combine_payload_bytes=938901504 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=939360256 ingress_bytes=1467383808 link_bytes=1467383808 token_bytes=1878261760 weight_movement_bytes=528482304 total_nvlink_bytes=2406744064 weight_to_token_ratio=0.281368 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.216736809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=932528128 dispatch_meta_bytes=458760 combine_payload_bytes=932528128 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=932986888 ingress_bytes=1461010432 link_bytes=1461010432 token_bytes=1865515016 weight_movement_bytes=528482304 total_nvlink_bytes=2393997320 weight_to_token_ratio=0.283290 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.181048563e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130600 payload_rows_remote=114228 dup_rows=472 dup_rows_remote=306 dedup_ratio=0.003601 router_tokens_per_expert=[8186, 8183, 4, 2, 0, 8080, 8149, 8117, 4, 4, 1, 8184, 0, 8, 0, 6, 5, 0, 5, 25, 1, 4, 8184, 5, 8181, 6, 2, 1, 36, 0, 0, 153] payload_rows_per_dst=[16232, 16228, 16382, 16256, 16384, 16370, 16376, 16372] dup_rows_per_dst=[0, 0, 4, 222, 58, 10, 12, 166] combine_rows_per_src=[16232, 16228, 16382, 16256, 16384, 16370, 16376, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=932413440 dispatch_meta_bytes=458752 combine_payload_bytes=932413440 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=932872192 ingress_bytes=1284734976 link_bytes=1284734976 token_bytes=1865285632 weight_movement_bytes=352321536 total_nvlink_bytes=2217607168 weight_to_token_ratio=0.188883 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.193979591e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=935165952 dispatch_meta_bytes=458904 combine_payload_bytes=935165952 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=935624856 ingress_bytes=1639809024 link_bytes=1639809024 token_bytes=1870790808 weight_movement_bytes=704643072 total_nvlink_bytes=2575433880 weight_to_token_ratio=0.376655 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.182246044e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130520 payload_rows_remote=114688 dup_rows=552 dup_rows_remote=552 dedup_ratio=0.004211 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 8176, 7916, 8192, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 290, 0, 0, 1] payload_rows_per_dst=[16384, 15832, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 550] combine_rows_per_src=[16384, 15832, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128282 payload_rows_remote=111898 dup_rows=2790 dup_rows_remote=2790 dedup_ratio=0.021286 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8172, 0, 0, 18, 8192, 2, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16368, 16344, 16384, 16328, 13706, 16384] dup_rows_per_dst=[0, 78, 0, 0, 2708, 4, 0, 0] combine_rows_per_src=[16384, 16384, 16368, 16344, 16384, 16328, 13706, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=935755776 dispatch_meta_bytes=458136 combine_payload_bytes=935755776 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936213912 ingress_bytes=1288077312 link_bytes=1288077312 token_bytes=1871969688 weight_movement_bytes=352321536 total_nvlink_bytes=2224291224 weight_to_token_ratio=0.188209 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.212695277e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130502 payload_rows_remote=114120 dup_rows=570 dup_rows_remote=562 dedup_ratio=0.004349 router_tokens_per_expert=[8188, 8187, 0, 1, 0, 8068, 7935, 8169, 2, 2, 3, 8188, 0, 4, 0, 2, 3, 1, 2, 10, 1, 3, 8188, 3, 8187, 3, 2, 0, 276, 0, 0, 108] payload_rows_per_dst=[16376, 15870, 16384, 16346, 16384, 16382, 16382, 16378] dup_rows_per_dst=[0, 0, 6, 4, 24, 8, 2, 526] combine_rows_per_src=[16376, 15870, 16384, 16346, 16384, 16382, 16382, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 4, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=460960 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=939985056 ingress_bytes=2348810240 link_bytes=2348810240 token_bytes=1879509152 weight_movement_bytes=1409286144 total_nvlink_bytes=3288795296 weight_to_token_ratio=0.749816 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315235690e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130694 payload_rows_remote=114310 dup_rows=378 dup_rows_remote=378 dedup_ratio=0.002884 router_tokens_per_expert=[8192, 0, 8192, 0, 8173, 0, 0, 0, 172, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 8022, 0, 0, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16346, 16384, 16384, 16384, 16384, 16044, 16384] dup_rows_per_dst=[0, 0, 344, 0, 0, 34, 0, 0] combine_rows_per_src=[16384, 16346, 16384, 16384, 16384, 16384, 16044, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130860 payload_rows_remote=114476 dup_rows=212 dup_rows_remote=204 dedup_ratio=0.001617 router_tokens_per_expert=[1, 2, 2, 1, 8190, 8190, 8188, 8190, 1, 1, 3, 1, 2, 1, 0, 0, 0, 8190, 0, 8094, 0, 0, 2, 8190, 33, 1, 1, 62, 8190, 0, 0, 0] payload_rows_per_dst=[16384, 16376, 16384, 16384, 16188, 16380, 16384, 16380] dup_rows_per_dst=[8, 0, 8, 2, 0, 4, 190, 0] combine_rows_per_src=[16384, 16376, 16384, 16384, 16188, 16380, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=916668416 dispatch_meta_bytes=458752 combine_payload_bytes=916668416 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=917127168 ingress_bytes=1268989952 link_bytes=1268989952 token_bytes=1833795584 weight_movement_bytes=352321536 total_nvlink_bytes=2186117120 weight_to_token_ratio=0.192127 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.105814029e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=934871040 dispatch_meta_bytes=458728 combine_payload_bytes=934871040 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=935329768 ingress_bytes=1815674880 link_bytes=1815674880 token_bytes=1870200808 weight_movement_bytes=880803840 total_nvlink_bytes=2751004648 weight_to_token_ratio=0.470968 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.016702143e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131050 payload_rows_remote=114666 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000168 router_tokens_per_expert=[0, 0, 0, 0, 8191, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 8192, 0, 0, 10, 8192, 0, 0, 0, 0, 8182, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16364] dup_rows_per_dst=[0, 0, 0, 2, 0, 20, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129690 payload_rows_remote=113310 dup_rows=1382 dup_rows_remote=1368 dedup_ratio=0.010544 router_tokens_per_expert=[13, 0, 19, 8, 8183, 8180, 8170, 8184, 1, 8, 0, 0, 10, 7, 0, 0, 14, 8181, 0, 7535, 0, 0, 7, 8177, 3, 2, 4, 645, 8185, 0, 0, 0] payload_rows_per_dst=[16384, 16340, 16382, 16380, 15098, 16354, 16382, 16370] dup_rows_per_dst=[62, 0, 4, 14, 0, 14, 1288, 0] combine_rows_per_src=[16384, 16340, 16382, 16380, 15098, 16354, 16382, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936427520 dispatch_meta_bytes=458752 combine_payload_bytes=936427520 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936886272 ingress_bytes=1288749056 link_bytes=1288749056 token_bytes=1873313792 weight_movement_bytes=352321536 total_nvlink_bytes=2225635328 weight_to_token_ratio=0.188074 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.216456763e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=937787392 dispatch_meta_bytes=458720 combine_payload_bytes=937787392 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938246112 ingress_bytes=1290108928 link_bytes=1290108928 token_bytes=1876033504 weight_movement_bytes=352321536 total_nvlink_bytes=2228355040 weight_to_token_ratio=0.187801 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224071479e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130142 payload_rows_remote=113758 dup_rows=930 dup_rows_remote=876 dedup_ratio=0.007095 router_tokens_per_expert=[11, 0, 12, 6, 8186, 8182, 8174, 8186, 0, 7, 1, 0, 7, 4, 0, 0, 3, 8183, 0, 7813, 0, 0, 0, 8182, 5, 5, 2, 375, 8186, 0, 6, 0] payload_rows_per_dst=[16268, 16348, 16380, 16384, 15632, 16364, 16382, 16384] dup_rows_per_dst=[40, 30, 8, 8, 32, 0, 758, 54] combine_rows_per_src=[16268, 16348, 16380, 16384, 15632, 16364, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=458752 combine_payload_bytes=939343872 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939802624 ingress_bytes=1291665408 link_bytes=1291665408 token_bytes=1879146496 weight_movement_bytes=352321536 total_nvlink_bytes=2231468032 weight_to_token_ratio=0.187490 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232787117e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=928235520 dispatch_meta_bytes=458712 combine_payload_bytes=928235520 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928694232 ingress_bytes=1280557056 link_bytes=1280557056 token_bytes=1856929752 weight_movement_bytes=352321536 total_nvlink_bytes=2209251288 weight_to_token_ratio=0.189733 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.170584983e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114680 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8189, 0, 0, 4, 8192, 0, 0, 0, 0, 8191, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 8, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=931905536 dispatch_meta_bytes=458536 combine_payload_bytes=931905536 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=932364072 ingress_bytes=1108066304 link_bytes=1108066304 token_bytes=1864269608 weight_movement_bytes=176160768 total_nvlink_bytes=2040430376 weight_to_token_ratio=0.094493 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.204708773e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131034 payload_rows_remote=114650 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000290 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 8174, 21] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16372, 16384, 16362] dup_rows_per_dst=[0, 22, 8, 8, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16372, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130572 payload_rows_remote=114200 dup_rows=500 dup_rows_remote=486 dedup_ratio=0.003815 router_tokens_per_expert=[6, 0, 5, 3, 8184, 8187, 8182, 8189, 1, 3, 0, 0, 4, 3, 0, 5, 2, 8188, 0, 7969, 0, 0, 7, 8186, 3, 1, 2, 221, 8182, 0, 3, 0] payload_rows_per_dst=[16374, 16364, 16382, 16384, 15942, 16372, 16384, 16370] dup_rows_per_dst=[18, 0, 4, 18, 0, 14, 446, 0] combine_rows_per_src=[16374, 16364, 16382, 16384, 15942, 16372, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=939917312 ingress_bytes=2701066240 link_bytes=2701066240 token_bytes=1879375872 weight_movement_bytes=1761607680 total_nvlink_bytes=3640983552 weight_to_token_ratio=0.937337 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.512484346e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114680 dup_rows=10 dup_rows_remote=8 dedup_ratio=0.000076 router_tokens_per_expert=[1, 0, 0, 2, 1, 0, 8191, 2, 1, 1, 8190, 8190, 1, 0, 8190, 1, 0, 0, 0, 2, 1, 8190, 0, 0, 8185, 1, 1, 5, 0, 8190, 8162, 28] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16382, 16382, 16380] dup_rows_per_dst=[2, 4, 2, 0, 0, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16382, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130556 payload_rows_remote=114172 dup_rows=516 dup_rows_remote=516 dedup_ratio=0.003937 router_tokens_per_expert=[8192, 1, 89, 0, 8192, 0, 8192, 0, 7961, 8166, 0, 0, 8191, 0, 0, 135, 0, 0, 0, 0, 0, 32, 8192, 0, 0, 0, 8192, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15922, 16382, 16384, 16384, 16384, 16332] dup_rows_per_dst=[180, 0, 0, 270, 0, 64, 2, 0] combine_rows_per_src=[16384, 16384, 15922, 16382, 16384, 16384, 16384, 16332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131048 payload_rows_remote=114664 dup_rows=24 dup_rows_remote=18 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 3, 0, 8192, 8139, 60] payload_rows_per_dst=[16384, 16360, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 6, 14] combine_rows_per_src=[16384, 16360, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=458752 combine_payload_bytes=939212800 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939671552 ingress_bytes=1291534336 link_bytes=1291534336 token_bytes=1878884352 weight_movement_bytes=352321536 total_nvlink_bytes=2231205888 weight_to_token_ratio=0.187516 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232053169e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=935526400 dispatch_meta_bytes=458744 combine_payload_bytes=935526400 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=935985144 ingress_bytes=1464008704 link_bytes=1464008704 token_bytes=1871511544 weight_movement_bytes=528482304 total_nvlink_bytes=2399993848 weight_to_token_ratio=0.282383 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.197837635e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130972 payload_rows_remote=114600 dup_rows=100 dup_rows_remote=96 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 0, 12, 9, 0, 8184, 9, 16, 0, 8180, 8182, 2, 0, 8175, 11, 0, 0, 0, 10, 6, 8176, 1, 30, 8156, 3, 7, 0, 0, 8183, 8169, 15] payload_rows_per_dst=[16384, 16382, 16374, 16372, 16384, 16384, 16326, 16366] dup_rows_per_dst=[6, 22, 18, 4, 0, 42, 6, 2] combine_rows_per_src=[16384, 16382, 16374, 16372, 16384, 16384, 16326, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 3, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130848 payload_rows_remote=114476 dup_rows=224 dup_rows_remote=194 dedup_ratio=0.001709 router_tokens_per_expert=[3, 0, 0, 7, 7, 0, 8187, 6, 16, 2, 8184, 8185, 1, 0, 8182, 6, 0, 0, 0, 6, 5, 8177, 0, 20, 8164, 2, 5, 0, 0, 8185, 8186, 0] payload_rows_per_dst=[16356, 16362, 16326, 16376, 16382, 16336, 16338, 16372] dup_rows_per_dst=[6, 16, 26, 12, 4, 20, 110, 30] combine_rows_per_src=[16356, 16362, 16326, 16376, 16382, 16336, 16338, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=939917312 ingress_bytes=2348744704 link_bytes=2348744704 token_bytes=1879375872 weight_movement_bytes=1409286144 total_nvlink_bytes=3288662016 weight_to_token_ratio=0.749869 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315198993e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=458728 combine_payload_bytes=939327488 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939786216 ingress_bytes=1291649024 link_bytes=1291649024 token_bytes=1879113704 weight_movement_bytes=352321536 total_nvlink_bytes=2231435240 weight_to_token_ratio=0.187493 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232695374e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=935297024 dispatch_meta_bytes=458752 combine_payload_bytes=935297024 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935755776 ingress_bytes=1287618560 link_bytes=1287618560 token_bytes=1871052800 weight_movement_bytes=352321536 total_nvlink_bytes=2223374336 weight_to_token_ratio=0.188301 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.210126458e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938803200 dispatch_meta_bytes=458784 combine_payload_bytes=938803200 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939261984 ingress_bytes=1291124736 link_bytes=1291124736 token_bytes=1878065184 weight_movement_bytes=352321536 total_nvlink_bytes=2230386720 weight_to_token_ratio=0.187598 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229759580e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114688 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 8180, 12] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=937787392 dispatch_meta_bytes=458680 combine_payload_bytes=937787392 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=938246072 ingress_bytes=1818591232 link_bytes=1818591232 token_bytes=1876033464 weight_movement_bytes=880803840 total_nvlink_bytes=2756837304 weight_to_token_ratio=0.469503 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018335178e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131004 payload_rows_remote=114620 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.000519 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 8192, 0, 8192, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16316, 16384, 16384] dup_rows_per_dst=[0, 68, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16316, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131014 payload_rows_remote=114632 dup_rows=58 dup_rows_remote=56 dedup_ratio=0.000443 router_tokens_per_expert=[2, 0, 3, 4, 4, 0, 8178, 3, 11, 2, 8187, 8188, 0, 0, 8187, 3, 0, 0, 0, 3, 2, 8185, 0, 5, 8179, 1, 3, 0, 0, 8189, 8170, 27] payload_rows_per_dst=[16384, 16362, 16382, 16380, 16382, 16382, 16364, 16378] dup_rows_per_dst=[12, 8, 18, 0, 0, 2, 2, 16] combine_rows_per_src=[16384, 16362, 16382, 16380, 16382, 16382, 16364, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458760 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=939982856 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879506952 weight_movement_bytes=528482304 total_nvlink_bytes=2407989256 weight_to_token_ratio=0.281181 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131026 payload_rows_remote=114642 dup_rows=46 dup_rows_remote=12 dedup_ratio=0.000351 router_tokens_per_expert=[2, 0, 2, 8191, 0, 1, 0, 8189, 2, 13, 2, 2, 0, 1, 1, 0, 3, 0, 0, 0, 1, 8190, 0, 8189, 8189, 8190, 0, 8190, 1, 8177, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16382, 16382, 16380, 16378, 16356] dup_rows_per_dst=[6, 0, 34, 2, 4, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16382, 16382, 16380, 16378, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 8192, 0, 8192, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130580 payload_rows_remote=114200 dup_rows=492 dup_rows_remote=476 dedup_ratio=0.003754 router_tokens_per_expert=[13, 0, 0, 8182, 0, 8, 0, 8168, 9, 187, 10, 10, 2, 10, 6, 0, 0, 0, 31, 11, 9, 8181, 0, 8181, 8160, 8180, 1, 8174, 5, 7996, 2, 0] payload_rows_per_dst=[16372, 16352, 16384, 16380, 16384, 16380, 16322, 16006] dup_rows_per_dst=[18, 0, 396, 16, 62, 0, 0, 0] combine_rows_per_src=[16372, 16352, 16384, 16380, 16384, 16380, 16322, 16006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938967040 dispatch_meta_bytes=458752 combine_payload_bytes=938967040 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=939425792 ingress_bytes=1291288576 link_bytes=1291288576 token_bytes=1878392832 weight_movement_bytes=352321536 total_nvlink_bytes=2230714368 weight_to_token_ratio=0.187565 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230677015e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939065344 dispatch_meta_bytes=458752 combine_payload_bytes=939065344 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939524096 ingress_bytes=1819869184 link_bytes=1819869184 token_bytes=1878589440 weight_movement_bytes=880803840 total_nvlink_bytes=2759393280 weight_to_token_ratio=0.468864 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019050778e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130564 payload_rows_remote=114180 dup_rows=508 dup_rows_remote=2 dedup_ratio=0.003876 router_tokens_per_expert=[0, 0, 8192, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 253, 0, 0, 8192, 8192, 2, 0, 7936, 8192, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 15876, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 0, 506, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 15876, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129764 payload_rows_remote=113602 dup_rows=1308 dup_rows_remote=564 dedup_ratio=0.009979 router_tokens_per_expert=[10, 0, 2, 8185, 0, 6, 0, 8176, 6, 112, 8, 8, 2, 9, 4, 0, 0, 0, 25, 6, 6, 8183, 0, 8184, 8162, 8183, 0, 8178, 5, 8074, 2, 0] payload_rows_per_dst=[16350, 16364, 15590, 16348, 16248, 16378, 16324, 16162] dup_rows_per_dst=[16, 0, 240, 16, 48, 84, 160, 744] combine_rows_per_src=[16350, 16364, 15590, 16348, 16248, 16378, 16324, 16162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=458616 combine_payload_bytes=939147264 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939605880 ingress_bytes=1291468800 link_bytes=1291468800 token_bytes=1878753144 weight_movement_bytes=352321536 total_nvlink_bytes=2231074680 weight_to_token_ratio=0.187529 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231686194e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=939982848 ingress_bytes=2348810240 link_bytes=2348810240 token_bytes=1879506944 weight_movement_bytes=1409286144 total_nvlink_bytes=3288793088 weight_to_token_ratio=0.749817 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315235690e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=935526400 dispatch_meta_bytes=458704 combine_payload_bytes=935526400 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935985104 ingress_bytes=1287847936 link_bytes=1287847936 token_bytes=1871511504 weight_movement_bytes=352321536 total_nvlink_bytes=2223833040 weight_to_token_ratio=0.188255 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.211410867e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 8192, 0, 8192, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=935362560 dispatch_meta_bytes=456728 combine_payload_bytes=935362560 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=935819288 ingress_bytes=1640005632 link_bytes=1640005632 token_bytes=1871181848 weight_movement_bytes=704643072 total_nvlink_bytes=2575824920 weight_to_token_ratio=0.376576 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.183346967e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=930627584 dispatch_meta_bytes=456664 combine_payload_bytes=930627584 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=931084248 ingress_bytes=1282949120 link_bytes=1282949120 token_bytes=1861711832 weight_movement_bytes=352321536 total_nvlink_bytes=2214033368 weight_to_token_ratio=0.189246 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.183979543e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129468 payload_rows_remote=113098 dup_rows=1604 dup_rows_remote=334 dedup_ratio=0.012238 router_tokens_per_expert=[0, 8192, 0, 8152, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 40, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16370, 16384, 16384, 16384, 16384, 15308, 16384, 15870] dup_rows_per_dst=[1270, 0, 258, 0, 54, 14, 8, 0] combine_rows_per_src=[16370, 16384, 16384, 16384, 16384, 15308, 16384, 15870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130904 payload_rows_remote=114522 dup_rows=168 dup_rows_remote=168 dedup_ratio=0.001282 router_tokens_per_expert=[6, 0, 1, 8188, 0, 3, 0, 8183, 3, 60, 4, 4, 0, 7, 2, 0, 0, 0, 13, 3, 3, 8187, 1, 8188, 8176, 8187, 0, 8183, 3, 8129, 2, 0] payload_rows_per_dst=[16382, 16372, 16384, 16382, 16382, 16382, 16352, 16268] dup_rows_per_dst=[8, 0, 124, 10, 26, 0, 0, 0] combine_rows_per_src=[16382, 16372, 16384, 16382, 16382, 16382, 16352, 16268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879506944 weight_movement_bytes=352321536 total_nvlink_bytes=2231828480 weight_to_token_ratio=0.187454 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130794 payload_rows_remote=114418 dup_rows=278 dup_rows_remote=278 dedup_ratio=0.002121 router_tokens_per_expert=[0, 8190, 5, 8057, 2, 2, 0, 19, 1, 0, 8187, 0, 2, 0, 0, 1, 8188, 1, 2, 0, 115, 3, 0, 0, 8190, 0, 8190, 1, 0, 0, 8190, 8190] payload_rows_per_dst=[16124, 16384, 16376, 16384, 16380, 16384, 16382, 16380] dup_rows_per_dst=[0, 42, 0, 2, 2, 232, 0, 0] combine_rows_per_src=[16124, 16384, 16376, 16384, 16380, 16384, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130914 payload_rows_remote=114530 dup_rows=158 dup_rows_remote=158 dedup_ratio=0.001205 router_tokens_per_expert=[0, 8192, 0, 8122, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 78, 0, 1, 0, 8192, 0, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16244, 16384, 16368, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 158, 0, 0] combine_rows_per_src=[16244, 16384, 16368, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130516 payload_rows_remote=114132 dup_rows=556 dup_rows_remote=548 dedup_ratio=0.004242 router_tokens_per_expert=[12, 8183, 43, 7952, 17, 13, 5, 88, 3, 0, 8134, 0, 9, 0, 0, 5, 8166, 7, 10, 0, 143, 14, 0, 0, 8174, 0, 8182, 9, 0, 9, 8178, 8180] payload_rows_per_dst=[16010, 16384, 16274, 16384, 16352, 16384, 16366, 16362] dup_rows_per_dst=[4, 222, 0, 8, 14, 296, 0, 12] combine_rows_per_src=[16010, 16384, 16274, 16384, 16352, 16384, 16366, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=926498816 dispatch_meta_bytes=453728 combine_payload_bytes=926498816 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=926952544 ingress_bytes=1631141888 link_bytes=1631141888 token_bytes=1853451360 weight_movement_bytes=704643072 total_nvlink_bytes=2558094432 weight_to_token_ratio=0.380179 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.133713701e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938164224 dispatch_meta_bytes=458760 combine_payload_bytes=938164224 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=938622984 ingress_bytes=2347450368 link_bytes=2347450368 token_bytes=1876787208 weight_movement_bytes=1409286144 total_nvlink_bytes=3286073352 weight_to_token_ratio=0.750904 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314474219e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129548 payload_rows_remote=113164 dup_rows=1524 dup_rows_remote=1524 dedup_ratio=0.011627 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 8182, 7430, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 772, 0, 0, 0] payload_rows_per_dst=[16384, 14860, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 1524] combine_rows_per_src=[16384, 14860, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130242 payload_rows_remote=113876 dup_rows=830 dup_rows_remote=272 dedup_ratio=0.006332 router_tokens_per_expert=[3, 8186, 30, 8105, 12, 10, 13, 71, 2, 0, 8152, 0, 7, 0, 2, 6, 8174, 4, 7, 0, 7, 10, 0, 0, 8179, 0, 8185, 6, 0, 0, 8182, 8183] payload_rows_per_dst=[16276, 15826, 16308, 16350, 16362, 16384, 16370, 16366] dup_rows_per_dst=[0, 192, 0, 16, 8, 22, 34, 558] combine_rows_per_src=[16276, 15826, 16308, 16350, 16362, 16384, 16370, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937312256 dispatch_meta_bytes=458784 combine_payload_bytes=937312256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937771040 ingress_bytes=1289633792 link_bytes=1289633792 token_bytes=1875083296 weight_movement_bytes=352321536 total_nvlink_bytes=2227404832 weight_to_token_ratio=0.187896 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.221410916e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938229760 dispatch_meta_bytes=458752 combine_payload_bytes=938229760 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938688512 ingress_bytes=1466712064 link_bytes=1466712064 token_bytes=1876918272 weight_movement_bytes=528482304 total_nvlink_bytes=2405400576 weight_to_token_ratio=0.281569 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.212975323e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=934969344 dispatch_meta_bytes=458720 combine_payload_bytes=934969344 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935428064 ingress_bytes=1287290880 link_bytes=1287290880 token_bytes=1870397408 weight_movement_bytes=352321536 total_nvlink_bytes=2222718944 weight_to_token_ratio=0.188367 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.208291586e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28434702976 prefetch_bytes=5284823040 reduce_bytes=5460983808 weight_movement_bytes=10745806848 egress_bytes=14221009536 ingress_bytes=24959500288 link_bytes=24959500288 total_nvlink_bytes=39180509824 weight_to_token_ratio=0.377912 prefetch_share_of_ingress=0.211736 dedup_ratio=0.048098 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=31 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.017874 gemm_inner_pad_upper_bound=22479 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.735173938e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131030 payload_rows_remote=114646 dup_rows=42 dup_rows_remote=42 dedup_ratio=0.000320 router_tokens_per_expert=[0, 8192, 0, 8173, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 21, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16346, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 42, 0, 0] combine_rows_per_src=[16346, 16384, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 2, 0, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=927039488 dispatch_meta_bytes=458752 combine_payload_bytes=927039488 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=927498240 ingress_bytes=1279361024 link_bytes=1279361024 token_bytes=1854537728 weight_movement_bytes=352321536 total_nvlink_bytes=2206859264 weight_to_token_ratio=0.189978 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.163887703e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29375271240 prefetch_bytes=4756340736 reduce_bytes=5989466112 weight_movement_bytes=10745806848 egress_bytes=14691291464 ingress_bytes=25429786624 link_bytes=25429786624 total_nvlink_bytes=40121078088 weight_to_token_ratio=0.365811 prefetch_share_of_ingress=0.187038 dedup_ratio=0.022754 prefetch_slots=54 unused_prefetch_slots=202 slot_utilization=0.210938 reduce_slots=34 remote_demand=27 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019295 gemm_inner_pad_upper_bound=20828 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.899761886e-05 prefetch_bytes_per_gemm_flop=1.664595170e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=932872192 dispatch_meta_bytes=456592 combine_payload_bytes=932872192 prefetch_bytes=0 reduce_bytes=1585446912 egress_bytes=933328784 ingress_bytes=2518319104 link_bytes=2518319104 token_bytes=1866200976 weight_movement_bytes=1585446912 total_nvlink_bytes=3451647888 weight_to_token_ratio=0.849559 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.410153578e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29198353488 prefetch_bytes=5108662272 reduce_bytes=3875536896 weight_movement_bytes=8984199168 egress_bytes=14602847312 ingress_bytes=23579705344 link_bytes=23579705344 total_nvlink_bytes=38182552656 weight_to_token_ratio=0.307695 prefetch_share_of_ingress=0.216655 dedup_ratio=0.029479 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=22 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=21209 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.252281705e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29420207672 prefetch_bytes=5108662272 reduce_bytes=2466250752 weight_movement_bytes=7574913024 egress_bytes=14713781816 ingress_bytes=22281338880 link_bytes=22281338880 total_nvlink_bytes=36995120696 weight_to_token_ratio=0.257473 prefetch_share_of_ingress=0.229280 dedup_ratio=0.022724 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=14 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019531 gemm_inner_pad_upper_bound=20320 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.797887315e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939180032 dispatch_meta_bytes=458752 combine_payload_bytes=939180032 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939638784 ingress_bytes=1291501568 link_bytes=1291501568 token_bytes=1878818816 weight_movement_bytes=352321536 total_nvlink_bytes=2231140352 weight_to_token_ratio=0.187523 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231869682e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130862 payload_rows_remote=114478 dup_rows=210 dup_rows_remote=116 dedup_ratio=0.001602 router_tokens_per_expert=[2, 8187, 16, 8113, 7, 6, 7, 35, 1, 0, 8167, 0, 4, 0, 0, 4, 8179, 2, 4, 0, 47, 5, 0, 0, 8185, 0, 8188, 3, 0, 0, 8187, 8187] payload_rows_per_dst=[16260, 16384, 16336, 16384, 16366, 16384, 16376, 16372] dup_rows_per_dst=[2, 102, 0, 8, 4, 94, 0, 0] combine_rows_per_src=[16260, 16384, 16336, 16384, 16366, 16384, 16376, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28532798104 prefetch_bytes=4580179968 reduce_bytes=5284823040 weight_movement_bytes=9865003008 egress_bytes=14270001816 ingress_bytes=24127799296 link_bytes=24127799296 total_nvlink_bytes=38397801112 weight_to_token_ratio=0.345743 prefetch_share_of_ingress=0.189830 dedup_ratio=0.045436 prefetch_slots=52 unused_prefetch_slots=204 slot_utilization=0.203125 reduce_slots=30 remote_demand=26 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020123 gemm_inner_pad_upper_bound=19685 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.444100289e-05 prefetch_bytes_per_gemm_flop=1.602943497e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29113733896 prefetch_bytes=5637144576 reduce_bytes=5989466112 weight_movement_bytes=11626610688 egress_bytes=14560514824 ingress_bytes=26179829760 link_bytes=26179829760 total_nvlink_bytes=40740344584 weight_to_token_ratio=0.399351 prefetch_share_of_ingress=0.215324 dedup_ratio=0.030465 prefetch_slots=64 unused_prefetch_slots=192 slot_utilization=0.250000 reduce_slots=34 remote_demand=32 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019295 gemm_inner_pad_upper_bound=20193 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.162257416e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937803776 dispatch_meta_bytes=458376 combine_payload_bytes=937803776 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938262152 ingress_bytes=1290125312 link_bytes=1290125312 token_bytes=1876065928 weight_movement_bytes=352321536 total_nvlink_bytes=2228387464 weight_to_token_ratio=0.187798 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224163222e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131014 payload_rows_remote=114688 dup_rows=58 dup_rows_remote=58 dedup_ratio=0.000443 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8163, 0, 0, 29, 8192, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16326, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 58, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16326, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29384609920 prefetch_bytes=3699376128 reduce_bytes=8103395328 weight_movement_bytes=11802771456 egress_bytes=14695960704 ingress_bytes=26491420672 link_bytes=26491420672 total_nvlink_bytes=41187381376 weight_to_token_ratio=0.401665 prefetch_share_of_ingress=0.139644 dedup_ratio=0.020799 prefetch_slots=42 unused_prefetch_slots=214 slot_utilization=0.164062 reduce_slots=46 remote_demand=21 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019768 gemm_inner_pad_upper_bound=20066 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.271306106e-05 prefetch_bytes_per_gemm_flop=1.294685133e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458984 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939983080 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879507176 weight_movement_bytes=704643072 total_nvlink_bytes=2584150248 weight_to_token_ratio=0.374908 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 8142, 50] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879506944 weight_movement_bytes=528482304 total_nvlink_bytes=2407989248 weight_to_token_ratio=0.281181 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 8192, 0, 8192, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879506944 weight_movement_bytes=352321536 total_nvlink_bytes=2231828480 weight_to_token_ratio=0.187454 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130854 payload_rows_remote=114470 dup_rows=218 dup_rows_remote=218 dedup_ratio=0.001663 router_tokens_per_expert=[0, 8192, 0, 8083, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 109, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16166, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 218, 0, 0] combine_rows_per_src=[16166, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937738240 dispatch_meta_bytes=458752 combine_payload_bytes=937738240 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938196992 ingress_bytes=1290059776 link_bytes=1290059776 token_bytes=1875935232 weight_movement_bytes=352321536 total_nvlink_bytes=2228256768 weight_to_token_ratio=0.187811 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223796248e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29376136600 prefetch_bytes=6165626880 reduce_bytes=3170893824 weight_movement_bytes=9336520704 egress_bytes=14691747224 ingress_bytes=24020910080 link_bytes=24020910080 total_nvlink_bytes=38712657304 weight_to_token_ratio=0.317827 prefetch_share_of_ingress=0.256677 dedup_ratio=0.024155 prefetch_slots=70 unused_prefetch_slots=186 slot_utilization=0.273438 reduce_slots=18 remote_demand=35 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018703 gemm_inner_pad_upper_bound=21209 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.406691852e-05 prefetch_bytes_per_gemm_flop=2.157808554e-05 anomalies=none - [2026-09-16 21:37:07] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 581.0 | throughput per GPU (TFLOP/s/GPU): 573.8 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.012994E+01 | loss scale: 1.0 | grad norm: 7.733 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105156 payload_rows_remote=91690 dup_rows=25916 dup_rows_remote=21030 dedup_ratio=0.197723 router_tokens_per_expert=[217, 8, 4610, 89, 1490, 2, 6800, 17, 3, 680, 0, 3195, 5000, 3647, 20, 509, 26, 93, 7050, 5, 0, 463, 2029, 31, 2, 6761, 8093, 4502, 4514, 875, 1877, 2928] payload_rows_per_dst=[14146, 14162, 14732, 13466, 14130, 16296, 9006, 9218] dup_rows_per_dst=[4730, 2456, 6546, 4886, 218, 4936, 2, 2142] combine_rows_per_src=[14146, 14162, 14732, 13466, 14130, 16296, 9006, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97944 payload_rows_remote=83014 dup_rows=33128 dup_rows_remote=26482 dedup_ratio=0.252747 router_tokens_per_expert=[273, 1, 5323, 54, 509, 0, 6695, 19, 0, 1025, 0, 3174, 4865, 4397, 2, 483, 17, 76, 7109, 0, 0, 708, 2033, 16, 0, 6221, 8109, 4466, 5137, 1455, 1548, 1821] payload_rows_per_dst=[14930, 15452, 14240, 13906, 14234, 16316, 722, 8144] dup_rows_per_dst=[6646, 7204, 6600, 5588, 170, 5416, 0, 1504] combine_rows_per_src=[14930, 15452, 14240, 13906, 14234, 16316, 722, 8144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105064 payload_rows_remote=89852 dup_rows=26008 dup_rows_remote=18072 dedup_ratio=0.198425 router_tokens_per_expert=[160, 0, 5404, 24, 935, 7, 6690, 17, 15, 990, 0, 3623, 5105, 4094, 5, 446, 37, 134, 7278, 2, 0, 478, 1337, 23, 1, 6946, 8132, 4371, 4812, 1068, 1630, 1772] payload_rows_per_dst=[14874, 13702, 15212, 14034, 14600, 16318, 8744, 7580] dup_rows_per_dst=[5926, 1596, 7936, 5266, 302, 3622, 0, 1360] combine_rows_per_src=[14874, 13702, 15212, 14034, 14600, 16318, 8744, 7580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=91588 payload_rows_remote=75228 dup_rows=39484 dup_rows_remote=18402 dedup_ratio=0.301239 router_tokens_per_expert=[215, 10, 5354, 51, 904, 1, 6630, 6, 6, 1127, 2, 2659, 4586, 4240, 18, 313, 26, 72, 7101, 2, 1, 250, 2008, 28, 2, 6021, 8100, 4684, 5518, 1181, 1985, 2435] payload_rows_per_dst=[10904, 13688, 6936, 9660, 15052, 4560, 16360, 14428] dup_rows_per_dst=[356, 1394, 652, 2204, 3964, 186, 21082, 9646] combine_rows_per_src=[10904, 13688, 6936, 9660, 15052, 4560, 16360, 14428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=102228 payload_rows_remote=88600 dup_rows=28844 dup_rows_remote=26478 dedup_ratio=0.220062 router_tokens_per_expert=[251, 5, 5692, 173, 1089, 6, 6209, 24, 6, 949, 6, 3104, 4333, 4967, 24, 584, 21, 126, 7508, 2, 0, 588, 1678, 35, 6, 6144, 8024, 4630, 4621, 1136, 1739, 1856] payload_rows_per_dst=[15086, 13628, 14276, 14332, 15050, 16178, 5688, 7990] dup_rows_per_dst=[6398, 2366, 6142, 7730, 264, 4472, 0, 1472] combine_rows_per_src=[15086, 13628, 14276, 14332, 15050, 16178, 5688, 7990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=84610 payload_rows_remote=69194 dup_rows=46462 dup_rows_remote=32308 dedup_ratio=0.354477 router_tokens_per_expert=[82, 0, 5552, 10, 558, 0, 6635, 8, 1, 1086, 0, 2333, 5257, 4484, 2, 177, 1, 77, 6930, 0, 0, 235, 1610, 11, 0, 6030, 8119, 4622, 6100, 1249, 1956, 2411] payload_rows_per_dst=[11176, 13430, 6376, 2460, 15740, 3644, 16368, 15416] dup_rows_per_dst=[112, 972, 464, 36, 8790, 68, 21866, 14154] combine_rows_per_src=[11176, 13430, 6376, 2460, 15740, 3644, 16368, 15416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=103282 payload_rows_remote=88270 dup_rows=27790 dup_rows_remote=24022 dedup_ratio=0.212021 router_tokens_per_expert=[287, 0, 4702, 53, 463, 2, 7064, 7, 1, 1704, 0, 2206, 4628, 4866, 8, 282, 5, 40, 6996, 0, 0, 292, 1897, 12, 0, 6305, 8070, 4372, 5720, 2040, 1833, 1681] payload_rows_per_dst=[9782, 14236, 7674, 14126, 13998, 15012, 14290, 14164] dup_rows_per_dst=[302, 836, 1376, 5442, 84, 3768, 7598, 8384] combine_rows_per_src=[9782, 14236, 7674, 14126, 13998, 15012, 14290, 14164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 1, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=751124480 dispatch_meta_bytes=450880 combine_payload_bytes=751124480 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=751575360 ingress_bytes=1808089088 link_bytes=1808089088 token_bytes=1502699840 weight_movement_bytes=1056964608 total_nvlink_bytes=2559664448 weight_to_token_ratio=0.703377 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.012454416e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=680050688 dispatch_meta_bytes=437984 combine_payload_bytes=680050688 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=680488672 ingress_bytes=1032372224 link_bytes=1032372224 token_bytes=1360539360 weight_movement_bytes=352321536 total_nvlink_bytes=1712860896 weight_to_token_ratio=0.258957 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=20992 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.780853521e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=736067584 dispatch_meta_bytes=431696 combine_payload_bytes=736067584 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=736499280 ingress_bytes=1088389120 link_bytes=1088389120 token_bytes=1472566864 weight_movement_bytes=352321536 total_nvlink_bytes=1824888400 weight_to_token_ratio=0.239257 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=16896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.094524756e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=616267776 dispatch_meta_bytes=374520 combine_payload_bytes=616267776 prefetch_bytes=176160768 reduce_bytes=1233125376 egress_bytes=616642296 ingress_bytes=2025553920 link_bytes=2025553920 token_bytes=1232910072 weight_movement_bytes=1409286144 total_nvlink_bytes=2642196216 weight_to_token_ratio=1.143057 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=15744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.134225644e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105946 payload_rows_remote=91632 dup_rows=25126 dup_rows_remote=24964 dedup_ratio=0.191696 router_tokens_per_expert=[227, 1, 4943, 121, 978, 0, 6962, 35, 9, 1040, 4, 3341, 4203, 4320, 4, 441, 8, 92, 7138, 0, 0, 303, 1736, 21, 0, 6585, 8080, 4649, 5301, 965, 1882, 2147] payload_rows_per_dst=[12550, 14156, 14826, 13288, 14314, 16292, 9320, 11200] dup_rows_per_dst=[3154, 1794, 7084, 4648, 162, 3988, 26, 4270] combine_rows_per_src=[12550, 14156, 14826, 13288, 14314, 16292, 9320, 11200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=725811200 dispatch_meta_bytes=460312 combine_payload_bytes=725811200 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=726271512 ingress_bytes=1078132736 link_bytes=1078132736 token_bytes=1452082712 weight_movement_bytes=352321536 total_nvlink_bytes=1804404248 weight_to_token_ratio=0.242632 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=28032 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.037093287e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=566837248 dispatch_meta_bytes=406008 combine_payload_bytes=566837248 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=567243256 ingress_bytes=1271480320 link_bytes=1271480320 token_bytes=1134080504 weight_movement_bytes=704643072 total_nvlink_bytes=1838723576 weight_to_token_ratio=0.621334 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=11008 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.119759050e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=723107840 dispatch_meta_bytes=449168 combine_payload_bytes=723107840 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=723557008 ingress_bytes=1075429376 link_bytes=1075429376 token_bytes=1446664848 weight_movement_bytes=352321536 total_nvlink_bytes=1798986384 weight_to_token_ratio=0.243541 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=24448 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.021955599e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130952 payload_rows_remote=114572 dup_rows=120 dup_rows_remote=92 dedup_ratio=0.000916 router_tokens_per_expert=[8172, 0, 8189, 0, 8181, 3, 1, 0, 8181, 8187, 0, 0, 1, 10, 8, 8185, 8182, 2, 0, 1, 3, 1, 3, 0, 9, 9, 19, 6, 2, 6, 0, 8175] payload_rows_per_dst=[16342, 16368, 16362, 16380, 16368, 16384, 16384, 16364] dup_rows_per_dst=[0, 2, 0, 28, 2, 10, 76, 2] combine_rows_per_src=[16342, 16368, 16362, 16380, 16368, 16384, 16384, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130862 payload_rows_remote=114478 dup_rows=210 dup_rows_remote=160 dedup_ratio=0.001602 router_tokens_per_expert=[8189, 3, 8184, 2, 8192, 0, 0, 3, 8186, 8190, 0, 0, 0, 0, 5, 8186, 8187, 1, 3, 0, 3, 0, 0, 4, 0, 9, 6, 0, 0, 1, 1, 8181] payload_rows_per_dst=[16384, 16384, 16380, 16338, 16372, 16364, 16276, 16364] dup_rows_per_dst=[50, 22, 120, 0, 0, 4, 12, 2] combine_rows_per_src=[16384, 16384, 16380, 16338, 16372, 16364, 16276, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114680 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[8192, 0, 8192, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 1, 2, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 8179] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16376] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=750649344 dispatch_meta_bytes=466384 combine_payload_bytes=750649344 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=751115728 ingress_bytes=1102970880 link_bytes=1102970880 token_bytes=1501765072 weight_movement_bytes=352321536 total_nvlink_bytes=1854086608 weight_to_token_ratio=0.234605 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=29312 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.176176525e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131044 payload_rows_remote=114666 dup_rows=28 dup_rows_remote=20 dedup_ratio=0.000214 router_tokens_per_expert=[8188, 0, 8186, 3, 8188, 0, 1, 0, 8189, 8192, 0, 1, 4, 0, 1, 8188, 8190, 2, 3, 1, 0, 1, 0, 0, 0, 2, 2, 0, 3, 0, 0, 8191] payload_rows_per_dst=[16380, 16378, 16382, 16382, 16382, 16378, 16378, 16384] dup_rows_per_dst=[0, 0, 2, 4, 10, 0, 8, 4] combine_rows_per_src=[16380, 16378, 16382, 16382, 16382, 16378, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114652 dup_rows=40 dup_rows_remote=38 dedup_ratio=0.000305 router_tokens_per_expert=[8189, 0, 8192, 0, 8190, 0, 0, 1, 8188, 8192, 0, 0, 0, 8, 6, 8185, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 0, 0, 7, 0, 8175] payload_rows_per_dst=[16378, 16380, 16376, 16382, 16384, 16384, 16384, 16364] dup_rows_per_dst=[0, 2, 0, 16, 0, 8, 14, 0] combine_rows_per_src=[16378, 16380, 16376, 16382, 16384, 16384, 16384, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130934 payload_rows_remote=114564 dup_rows=138 dup_rows_remote=42 dedup_ratio=0.001053 router_tokens_per_expert=[8192, 0, 8192, 2, 8192, 0, 0, 0, 8192, 8189, 0, 0, 1, 0, 7, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8171] payload_rows_per_dst=[16380, 16384, 16302, 16352, 16384, 16384, 16378, 16370] dup_rows_per_dst=[4, 0, 2, 16, 18, 2, 0, 96] combine_rows_per_src=[16380, 16384, 16302, 16352, 16384, 16384, 16378, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938573824 dispatch_meta_bytes=458656 combine_payload_bytes=938573824 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=939032480 ingress_bytes=1819377664 link_bytes=1819377664 token_bytes=1877606304 weight_movement_bytes=880803840 total_nvlink_bytes=2758410144 weight_to_token_ratio=0.469110 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018775547e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=937803776 dispatch_meta_bytes=458552 combine_payload_bytes=937803776 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=938262328 ingress_bytes=1818607616 link_bytes=1818607616 token_bytes=1876066104 weight_movement_bytes=880803840 total_nvlink_bytes=2756869944 weight_to_token_ratio=0.469495 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018344353e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131044 payload_rows_remote=114660 dup_rows=28 dup_rows_remote=26 dedup_ratio=0.000214 router_tokens_per_expert=[8190, 0, 8191, 1, 8190, 1, 1, 0, 8191, 8189, 0, 0, 0, 1, 0, 8192, 8191, 0, 1, 0, 2, 0, 0, 0, 1, 8, 3, 0, 1, 6, 0, 8176] payload_rows_per_dst=[16382, 16382, 16378, 16384, 16384, 16384, 16384, 16366] dup_rows_per_dst=[2, 2, 0, 2, 0, 2, 20, 0] combine_rows_per_src=[16382, 16382, 16378, 16384, 16384, 16384, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939917312 ingress_bytes=1996423168 link_bytes=1996423168 token_bytes=1879375872 weight_movement_bytes=1056964608 total_nvlink_bytes=2936340480 weight_to_token_ratio=0.562402 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117913639e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=458744 combine_payload_bytes=939343872 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939802616 ingress_bytes=1291665408 link_bytes=1291665408 token_bytes=1879146488 weight_movement_bytes=352321536 total_nvlink_bytes=2231468024 weight_to_token_ratio=0.187490 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232787117e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=458760 combine_payload_bytes=939229184 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939687944 ingress_bytes=1291550720 link_bytes=1291550720 token_bytes=1878917128 weight_movement_bytes=352321536 total_nvlink_bytes=2231238664 weight_to_token_ratio=0.187513 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232144912e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938508288 dispatch_meta_bytes=458424 combine_payload_bytes=938508288 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938966712 ingress_bytes=1290829824 link_bytes=1290829824 token_bytes=1877475000 weight_movement_bytes=352321536 total_nvlink_bytes=2229796536 weight_to_token_ratio=0.187657 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228108196e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=458744 combine_payload_bytes=939294720 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939753464 ingress_bytes=1467777024 link_bytes=1467777024 token_bytes=1879048184 weight_movement_bytes=528482304 total_nvlink_bytes=2407530488 weight_to_token_ratio=0.281250 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218938654e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114672 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000122 router_tokens_per_expert=[8192, 0, 8192, 1, 8192, 0, 0, 0, 8192, 8186, 0, 0, 5, 0, 2, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16372, 16380] dup_rows_per_dst=[2, 0, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16372, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130840 payload_rows_remote=114456 dup_rows=232 dup_rows_remote=102 dedup_ratio=0.001770 router_tokens_per_expert=[34, 8191, 8192, 0, 3, 0, 8, 0, 8181, 8192, 8184, 1, 8188, 0, 0, 6, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1, 8186, 0, 8158, 0, 0] payload_rows_per_dst=[16384, 16348, 16384, 16376, 16368, 16290, 16374, 16316] dup_rows_per_dst=[130, 6, 96, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16348, 16384, 16376, 16368, 16290, 16374, 16316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130986 payload_rows_remote=114602 dup_rows=86 dup_rows_remote=82 dedup_ratio=0.000656 router_tokens_per_expert=[9, 8189, 8181, 3, 0, 2, 5, 0, 8179, 8187, 8181, 1, 8189, 1, 4, 0, 4, 4, 0, 3, 0, 0, 17, 3, 3, 0, 3, 8177, 0, 8177, 2, 12] payload_rows_per_dst=[16384, 16374, 16360, 16384, 16382, 16374, 16364, 16364] dup_rows_per_dst=[18, 2, 0, 4, 14, 28, 2, 18] combine_rows_per_src=[16384, 16374, 16360, 16384, 16382, 16374, 16364, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131034 payload_rows_remote=114678 dup_rows=38 dup_rows_remote=38 dedup_ratio=0.000290 router_tokens_per_expert=[6, 8192, 8191, 0, 0, 0, 9, 0, 8178, 8192, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8188, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16382, 16356, 16384, 16384, 16384, 16376, 16384] dup_rows_per_dst=[12, 18, 0, 0, 0, 8, 0, 0] combine_rows_per_src=[16384, 16382, 16356, 16384, 16384, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=937623552 dispatch_meta_bytes=458232 combine_payload_bytes=937623552 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=938081784 ingress_bytes=2699231232 link_bytes=2699231232 token_bytes=1875705336 weight_movement_bytes=1761607680 total_nvlink_bytes=3637313016 weight_to_token_ratio=0.939171 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.511456818e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131010 payload_rows_remote=114628 dup_rows=62 dup_rows_remote=52 dedup_ratio=0.000473 router_tokens_per_expert=[17, 8192, 8191, 0, 0, 0, 4, 1, 8174, 8192, 8187, 0, 8189, 0, 0, 4, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 8182, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 16382, 16348, 16382, 16384, 16384, 16364, 16382] dup_rows_per_dst=[34, 10, 0, 4, 0, 14, 0, 0] combine_rows_per_src=[16384, 16382, 16348, 16382, 16384, 16384, 16364, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131024 payload_rows_remote=114642 dup_rows=48 dup_rows_remote=46 dedup_ratio=0.000366 router_tokens_per_expert=[9, 8184, 8192, 0, 1, 1, 2, 1, 8178, 8191, 8187, 0, 8188, 1, 0, 2, 0, 0, 1, 0, 0, 0, 19, 1, 1, 0, 2, 8189, 1, 8185, 0, 0] payload_rows_per_dst=[16380, 16384, 16356, 16382, 16384, 16384, 16382, 16372] dup_rows_per_dst=[6, 10, 0, 0, 0, 30, 2, 0] combine_rows_per_src=[16380, 16384, 16356, 16382, 16384, 16384, 16382, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=938819584 dispatch_meta_bytes=458736 combine_payload_bytes=938819584 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=939278320 ingress_bytes=1291141120 link_bytes=1291141120 token_bytes=1878097904 weight_movement_bytes=352321536 total_nvlink_bytes=2230419440 weight_to_token_ratio=0.187595 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229851323e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=458752 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939851776 ingress_bytes=1467875328 link_bytes=1467875328 token_bytes=1879244800 weight_movement_bytes=528482304 total_nvlink_bytes=2407727104 weight_to_token_ratio=0.281221 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219489115e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130852 payload_rows_remote=114468 dup_rows=220 dup_rows_remote=124 dedup_ratio=0.001678 router_tokens_per_expert=[9, 8187, 8192, 0, 0, 0, 0, 0, 8187, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8192, 0, 0] payload_rows_per_dst=[16176, 16384, 16372, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8, 0, 54, 0, 2, 2, 58, 96] combine_rows_per_src=[16176, 16384, 16372, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131036 payload_rows_remote=114654 dup_rows=36 dup_rows_remote=14 dedup_ratio=0.000275 router_tokens_per_expert=[1, 8188, 8191, 1, 1, 0, 1, 0, 8186, 8190, 8190, 1, 8191, 1, 1, 1, 1, 1, 0, 1, 1, 0, 11, 0, 0, 0, 0, 8191, 0, 8185, 0, 1] payload_rows_per_dst=[16378, 16382, 16374, 16384, 16382, 16382, 16382, 16372] dup_rows_per_dst=[2, 4, 0, 4, 4, 22, 0, 0] combine_rows_per_src=[16378, 16382, 16374, 16384, 16382, 16382, 16382, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458864 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=939901040 ingress_bytes=2172567552 link_bytes=2172567552 token_bytes=1879343216 weight_movement_bytes=1233125376 total_nvlink_bytes=3112468592 weight_to_token_ratio=0.656147 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216547142e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=458720 combine_payload_bytes=939032576 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939491296 ingress_bytes=1291354112 link_bytes=1291354112 token_bytes=1878523872 weight_movement_bytes=352321536 total_nvlink_bytes=2230845408 weight_to_token_ratio=0.187552 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231043989e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=458752 combine_payload_bytes=939147264 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939606016 ingress_bytes=1291468800 link_bytes=1291468800 token_bytes=1878753280 weight_movement_bytes=352321536 total_nvlink_bytes=2231074816 weight_to_token_ratio=0.187529 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231686194e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=937721856 dispatch_meta_bytes=458368 combine_payload_bytes=937721856 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938180224 ingress_bytes=1290043392 link_bytes=1290043392 token_bytes=1875902080 weight_movement_bytes=352321536 total_nvlink_bytes=2228223616 weight_to_token_ratio=0.187814 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223704505e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=458672 combine_payload_bytes=939245568 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939704240 ingress_bytes=1291567104 link_bytes=1291567104 token_bytes=1878949808 weight_movement_bytes=352321536 total_nvlink_bytes=2231271344 weight_to_token_ratio=0.187510 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232236656e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105102 payload_rows_remote=88718 dup_rows=25970 dup_rows_remote=25774 dedup_ratio=0.198135 router_tokens_per_expert=[0, 8142, 3, 0, 0, 0, 1380, 0, 2893, 1, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 8185, 8192, 0, 0, 42, 8192, 8192, 0, 848, 0, 0, 3083, 1] payload_rows_per_dst=[16384, 16380, 16384, 16384, 9324, 16178, 7902, 6166] dup_rows_per_dst=[196, 2760, 22158, 0, 0, 96, 758, 2] combine_rows_per_src=[16384, 16380, 16384, 16384, 9324, 16178, 7902, 6166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117518 payload_rows_remote=101134 dup_rows=13554 dup_rows_remote=13552 dedup_ratio=0.103409 router_tokens_per_expert=[5, 8164, 23, 5, 2, 0, 3011, 0, 1422, 1, 4, 8186, 1, 0, 4, 0, 8186, 0, 6, 8187, 8186, 2, 0, 0, 8188, 8185, 0, 234, 0, 4, 3523, 7] payload_rows_per_dst=[16378, 16380, 2854, 16384, 16384, 16376, 16380, 16382] dup_rows_per_dst=[16, 6018, 0, 2, 2, 0, 458, 7058] combine_rows_per_src=[16378, 16380, 2854, 16384, 16384, 16376, 16380, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131060 payload_rows_remote=114676 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000092 router_tokens_per_expert=[4, 8192, 8192, 0, 0, 0, 1, 0, 8186, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16372, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8, 2, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16372, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=109328 payload_rows_remote=96642 dup_rows=21744 dup_rows_remote=10628 dedup_ratio=0.165894 router_tokens_per_expert=[0, 8168, 1, 0, 0, 0, 3938, 0, 1262, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 8190, 0, 0, 16, 8192, 8192, 0, 745, 0, 0, 2256, 0] payload_rows_per_dst=[16336, 16384, 12686, 16384, 4696, 16380, 16384, 10078] dup_rows_per_dst=[2, 7876, 11116, 0, 0, 32, 1490, 1228] combine_rows_per_src=[16336, 16384, 12686, 16384, 4696, 16380, 16384, 10078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=726777856 dispatch_meta_bytes=457968 combine_payload_bytes=726777856 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=727235824 ingress_bytes=1079099392 link_bytes=1079099392 token_bytes=1454013680 weight_movement_bytes=352321536 total_nvlink_bytes=1806335216 weight_to_token_ratio=0.242310 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.042506157e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=102184 payload_rows_remote=85800 dup_rows=28888 dup_rows_remote=23258 dedup_ratio=0.220398 router_tokens_per_expert=[0, 8160, 1, 0, 0, 0, 2815, 0, 2541, 1, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 8192, 8190, 0, 0, 23, 8192, 8192, 0, 873, 0, 0, 1971, 2] payload_rows_per_dst=[16322, 16384, 16384, 16384, 0, 16380, 16384, 3946] dup_rows_per_dst=[0, 5630, 21466, 0, 0, 46, 1746, 0] combine_rows_per_src=[16322, 16384, 16384, 16384, 0, 16380, 16384, 3946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=828489728 dispatch_meta_bytes=458744 combine_payload_bytes=828489728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=828948472 ingress_bytes=1180811264 link_bytes=1180811264 token_bytes=1657438200 weight_movement_bytes=352321536 total_nvlink_bytes=2009759736 weight_to_token_ratio=0.212570 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.612050184e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108102 payload_rows_remote=91720 dup_rows=22970 dup_rows_remote=22254 dedup_ratio=0.175247 router_tokens_per_expert=[0, 8162, 6, 0, 0, 0, 2054, 2, 2869, 6, 0, 8187, 0, 0, 0, 0, 8186, 0, 0, 8188, 8192, 0, 0, 13, 8190, 8192, 0, 358, 0, 0, 2927, 4] payload_rows_per_dst=[16336, 4110, 5748, 16382, 16376, 16384, 16382, 16384] dup_rows_per_dst=[0, 2, 2, 0, 16372, 26, 716, 5852] combine_rows_per_src=[16336, 4110, 5748, 16382, 16376, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=106524 payload_rows_remote=90140 dup_rows=24548 dup_rows_remote=18158 dedup_ratio=0.187286 router_tokens_per_expert=[0, 8178, 2, 0, 0, 0, 932, 0, 3191, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 7, 8192, 8192, 0, 879, 0, 0, 3195, 0] payload_rows_per_dst=[16360, 1864, 6382, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 16384, 14, 1760, 6390] combine_rows_per_src=[16360, 1864, 6382, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458752 combine_payload_bytes=939425792 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939884544 ingress_bytes=1467908096 link_bytes=1467908096 token_bytes=1879310336 weight_movement_bytes=528482304 total_nvlink_bytes=2407792640 weight_to_token_ratio=0.281211 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219672603e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119784 payload_rows_remote=103422 dup_rows=11288 dup_rows_remote=11278 dedup_ratio=0.086121 router_tokens_per_expert=[1, 8179, 8, 1, 1, 0, 1470, 0, 3469, 3, 0, 8191, 0, 0, 1, 1, 8190, 0, 1, 8190, 8180, 1, 1, 4, 8190, 8191, 1, 496, 0, 1, 2765, 0] payload_rows_per_dst=[16366, 14580, 6944, 16384, 16384, 16362, 16380, 16384] dup_rows_per_dst=[12, 2358, 0, 2, 2382, 10, 994, 5530] combine_rows_per_src=[16366, 14580, 6944, 16384, 16384, 16362, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=791691264 dispatch_meta_bytes=429080 combine_payload_bytes=791691264 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=792120344 ingress_bytes=1496334336 link_bytes=1496334336 token_bytes=1583811608 weight_movement_bytes=704643072 total_nvlink_bytes=2288454680 weight_to_token_ratio=0.444903 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=11904 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.378847681e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=751370240 dispatch_meta_bytes=455896 combine_payload_bytes=751370240 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=751826136 ingress_bytes=1808334848 link_bytes=1808334848 token_bytes=1503196376 weight_movement_bytes=1056964608 total_nvlink_bytes=2560160984 weight_to_token_ratio=0.703145 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.012592031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=702873600 dispatch_meta_bytes=436232 combine_payload_bytes=702873600 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=703309832 ingress_bytes=1055195136 link_bytes=1055195136 token_bytes=1406183432 weight_movement_bytes=352321536 total_nvlink_bytes=1758504968 weight_to_token_ratio=0.250552 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=24192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.908652301e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=738426880 dispatch_meta_bytes=433192 combine_payload_bytes=738426880 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=738860072 ingress_bytes=1090748416 link_bytes=1090748416 token_bytes=1477286952 weight_movement_bytes=352321536 total_nvlink_bytes=1829608488 weight_to_token_ratio=0.238492 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.107735829e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130574 payload_rows_remote=114202 dup_rows=498 dup_rows_remote=498 dedup_ratio=0.003799 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 8192, 0, 8186, 6, 0, 8152, 0, 0, 0, 8192, 0, 8183, 0, 0, 0, 8185, 16, 8, 0, 9, 22, 8186, 8192, 0, 0, 2] payload_rows_per_dst=[16372, 16384, 16378, 16384, 16370, 16352, 15952, 16382] dup_rows_per_dst=[0, 12, 416, 0, 50, 12, 4, 4] combine_rows_per_src=[16372, 16384, 16378, 16384, 16370, 16352, 15952, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=847233024 dispatch_meta_bytes=458800 combine_payload_bytes=847233024 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=847691824 ingress_bytes=1551876096 link_bytes=1551876096 token_bytes=1694924848 weight_movement_bytes=704643072 total_nvlink_bytes=2399567920 weight_to_token_ratio=0.415737 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.689858353e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131000 payload_rows_remote=114618 dup_rows=72 dup_rows_remote=54 dedup_ratio=0.000549 router_tokens_per_expert=[5, 5, 6, 0, 0, 1, 8187, 3, 8187, 0, 7, 8179, 5, 6, 4, 8185, 1, 8186, 0, 0, 5, 8189, 0, 0, 0, 0, 7, 8187, 8179, 0, 2, 0] payload_rows_per_dst=[16384, 16380, 16362, 16382, 16372, 16384, 16374, 16362] dup_rows_per_dst=[22, 2, 10, 18, 2, 4, 14, 0] combine_rows_per_src=[16384, 16380, 16362, 16382, 16372, 16384, 16374, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=935542784 dispatch_meta_bytes=458800 combine_payload_bytes=935542784 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936001584 ingress_bytes=1287864320 link_bytes=1287864320 token_bytes=1871544368 weight_movement_bytes=352321536 total_nvlink_bytes=2223865904 weight_to_token_ratio=0.188252 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.211502611e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130982 payload_rows_remote=114680 dup_rows=90 dup_rows_remote=90 dedup_ratio=0.000687 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 8192, 0, 8192, 1, 0, 8150, 0, 0, 0, 8192, 0, 8190, 3, 0, 0, 8191, 0, 0, 0, 1, 39, 8191, 8192, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16302, 16384, 16380, 16382, 16382, 16384] dup_rows_per_dst=[0, 2, 0, 0, 6, 0, 80, 2] combine_rows_per_src=[16384, 16384, 16302, 16384, 16380, 16382, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123616 payload_rows_remote=107232 dup_rows=7456 dup_rows_remote=7456 dedup_ratio=0.056885 router_tokens_per_expert=[0, 8166, 16, 0, 0, 0, 1981, 0, 4474, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 10, 8192, 8192, 0, 285, 0, 0, 1452, 0] payload_rows_per_dst=[16364, 16384, 8948, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 3962, 0, 0, 0, 20, 570, 2904] combine_rows_per_src=[16364, 16384, 8948, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938950656 dispatch_meta_bytes=458688 combine_payload_bytes=938950656 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939409344 ingress_bytes=1291272192 link_bytes=1291272192 token_bytes=1878360000 weight_movement_bytes=352321536 total_nvlink_bytes=2230681536 weight_to_token_ratio=0.187569 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230585272e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131022 payload_rows_remote=114640 dup_rows=50 dup_rows_remote=20 dedup_ratio=0.000381 router_tokens_per_expert=[0, 0, 0, 3, 0, 1, 8192, 0, 8190, 6, 0, 8167, 0, 0, 0, 8192, 1, 8188, 0, 0, 0, 8187, 9, 3, 0, 4, 15, 8187, 8188, 0, 2, 1] payload_rows_per_dst=[16384, 16384, 16346, 16384, 16378, 16384, 16382, 16380] dup_rows_per_dst=[2, 2, 0, 0, 0, 14, 30, 2] combine_rows_per_src=[16384, 16384, 16346, 16384, 16378, 16384, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130924 payload_rows_remote=114540 dup_rows=148 dup_rows_remote=146 dedup_ratio=0.001129 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 8192, 0, 8187, 2, 0, 8127, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8184, 4, 0, 0, 8, 65, 8191, 8191, 0, 0, 0] payload_rows_per_dst=[16374, 16384, 16258, 16384, 16384, 16376, 16382, 16382] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 146, 0] combine_rows_per_src=[16374, 16384, 16258, 16384, 16384, 16376, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130984 payload_rows_remote=114600 dup_rows=88 dup_rows_remote=62 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 8189, 0, 0, 0, 8192, 0, 8188, 4, 0, 0, 8192, 0, 0, 0, 0, 2, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16380, 16364, 16378, 16364, 16346, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 4, 0, 2, 8, 46, 26] combine_rows_per_src=[16380, 16364, 16378, 16364, 16346, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=459080 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939917640 ingress_bytes=1644101632 link_bytes=1644101632 token_bytes=1879376200 weight_movement_bytes=704643072 total_nvlink_bytes=2584019272 weight_to_token_ratio=0.374935 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206282857e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=878444544 dispatch_meta_bytes=458752 combine_payload_bytes=878444544 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=878903296 ingress_bytes=1935409152 link_bytes=1935409152 token_bytes=1757347840 weight_movement_bytes=1056964608 total_nvlink_bytes=2814312448 weight_to_token_ratio=0.601454 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.083748337e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131036 payload_rows_remote=114652 dup_rows=36 dup_rows_remote=32 dedup_ratio=0.000275 router_tokens_per_expert=[1, 1, 1, 0, 0, 0, 8191, 1, 8190, 1, 0, 8179, 1, 1, 1, 8191, 0, 8190, 7, 0, 1, 8189, 4, 0, 0, 1, 5, 8190, 8189, 0, 0, 1] payload_rows_per_dst=[16382, 16384, 16360, 16384, 16382, 16384, 16382, 16378] dup_rows_per_dst=[4, 0, 0, 4, 12, 4, 10, 2] combine_rows_per_src=[16382, 16384, 16360, 16384, 16382, 16384, 16382, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=458640 combine_payload_bytes=939130880 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939589520 ingress_bytes=1643773952 link_bytes=1643773952 token_bytes=1878720400 weight_movement_bytes=704643072 total_nvlink_bytes=2583363472 weight_to_token_ratio=0.375065 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204447986e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938311680 dispatch_meta_bytes=458744 combine_payload_bytes=938311680 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=938770424 ingress_bytes=1642954752 link_bytes=1642954752 token_bytes=1877082104 weight_movement_bytes=704643072 total_nvlink_bytes=2581725176 weight_to_token_ratio=0.375393 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.199860808e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130886 payload_rows_remote=114502 dup_rows=186 dup_rows_remote=68 dedup_ratio=0.001419 router_tokens_per_expert=[0, 8, 8154, 0, 38, 0, 8192, 0, 0, 8185, 0, 0, 0, 6, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 8185, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16286, 16370, 16380, 16384, 16384, 16314, 16384] dup_rows_per_dst=[118, 12, 30, 12, 0, 14, 0, 0] combine_rows_per_src=[16384, 16286, 16370, 16380, 16384, 16384, 16314, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938803200 dispatch_meta_bytes=458648 combine_payload_bytes=938803200 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939261848 ingress_bytes=1291124736 link_bytes=1291124736 token_bytes=1878065048 weight_movement_bytes=352321536 total_nvlink_bytes=2230386584 weight_to_token_ratio=0.187598 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229759580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130970 payload_rows_remote=114598 dup_rows=102 dup_rows_remote=102 dedup_ratio=0.000778 router_tokens_per_expert=[0, 4, 8188, 9, 0, 7, 8185, 4, 6, 8186, 0, 3, 0, 0, 8186, 0, 4, 0, 6, 6, 8186, 8156, 0, 8186, 0, 7, 22, 8179, 0, 6, 0, 0] payload_rows_per_dst=[16376, 16384, 16384, 16372, 16384, 16312, 16374, 16384] dup_rows_per_dst=[26, 8, 6, 0, 20, 0, 42, 0] combine_rows_per_src=[16376, 16384, 16384, 16372, 16384, 16312, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=458736 combine_payload_bytes=939229184 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939687920 ingress_bytes=1467711488 link_bytes=1467711488 token_bytes=1878917104 weight_movement_bytes=528482304 total_nvlink_bytes=2407399408 weight_to_token_ratio=0.281270 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218571680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131052 payload_rows_remote=114668 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000153 router_tokens_per_expert=[0, 1, 8185, 0, 7, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8192, 8189, 0, 8192, 0, 0, 3, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16372, 16384, 16384, 16382, 16384, 16378, 16384, 16384] dup_rows_per_dst=[0, 14, 0, 0, 0, 0, 6, 0] combine_rows_per_src=[16372, 16384, 16384, 16382, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938000384 dispatch_meta_bytes=458280 combine_payload_bytes=938000384 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938458664 ingress_bytes=1290321920 link_bytes=1290321920 token_bytes=1876459048 weight_movement_bytes=352321536 total_nvlink_bytes=2228780584 weight_to_token_ratio=0.187759 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.225264145e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938786816 dispatch_meta_bytes=458800 combine_payload_bytes=938786816 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=939245616 ingress_bytes=1291108352 link_bytes=1291108352 token_bytes=1878032432 weight_movement_bytes=352321536 total_nvlink_bytes=2230353968 weight_to_token_ratio=0.187601 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229667836e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130938 payload_rows_remote=114562 dup_rows=134 dup_rows_remote=122 dedup_ratio=0.001022 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 8129, 0, 0, 0, 8192, 0, 8184, 10, 0, 0, 8192, 0, 0, 0, 0, 60, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16258, 16384, 16376, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 12, 0, 120, 0] combine_rows_per_src=[16384, 16384, 16258, 16384, 16376, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131004 payload_rows_remote=114636 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.000519 router_tokens_per_expert=[0, 8, 8168, 0, 27, 0, 8192, 0, 0, 8184, 0, 0, 0, 4, 8192, 0, 0, 2, 0, 0, 8192, 8191, 0, 8192, 0, 0, 0, 8184, 0, 0, 0, 0] payload_rows_per_dst=[16350, 16384, 16368, 16384, 16384, 16382, 16368, 16384] dup_rows_per_dst=[2, 54, 0, 8, 4, 0, 0, 0] combine_rows_per_src=[16350, 16384, 16368, 16384, 16384, 16382, 16368, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131028 payload_rows_remote=114644 dup_rows=44 dup_rows_remote=26 dedup_ratio=0.000336 router_tokens_per_expert=[0, 7, 8174, 9, 9, 0, 8192, 0, 0, 8192, 0, 0, 0, 2, 8191, 0, 0, 0, 0, 0, 8192, 8182, 0, 8192, 0, 0, 7, 8187, 0, 0, 0, 0] payload_rows_per_dst=[16366, 16384, 16384, 16382, 16384, 16364, 16380, 16384] dup_rows_per_dst=[14, 18, 0, 4, 0, 0, 8, 0] combine_rows_per_src=[16366, 16384, 16384, 16382, 16384, 16364, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458752 combine_payload_bytes=939360256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939819008 ingress_bytes=1291681792 link_bytes=1291681792 token_bytes=1879179264 weight_movement_bytes=352321536 total_nvlink_bytes=2231500800 weight_to_token_ratio=0.187487 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232878861e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130910 payload_rows_remote=114526 dup_rows=162 dup_rows_remote=162 dedup_ratio=0.001236 router_tokens_per_expert=[0, 0, 8171, 0, 14, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8191, 0, 8192, 0, 0, 9, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16342, 16300, 16382, 16382, 16354, 16382, 16384, 16384] dup_rows_per_dst=[0, 28, 0, 0, 0, 114, 20, 0] combine_rows_per_src=[16342, 16300, 16382, 16382, 16354, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938491904 dispatch_meta_bytes=458736 combine_payload_bytes=938491904 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=938950640 ingress_bytes=1466974208 link_bytes=1466974208 token_bytes=1877442544 weight_movement_bytes=528482304 total_nvlink_bytes=2405924848 weight_to_token_ratio=0.281491 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.214443220e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131028 payload_rows_remote=114670 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.000336 router_tokens_per_expert=[0, 4, 8183, 1, 6, 1, 8191, 1, 1, 8189, 0, 1, 0, 1, 8190, 0, 1, 1, 1, 1, 8191, 8179, 0, 8191, 0, 1, 9, 8191, 0, 1, 0, 0] payload_rows_per_dst=[16374, 16384, 16380, 16380, 16384, 16358, 16384, 16384] dup_rows_per_dst=[2, 14, 2, 2, 6, 0, 18, 0] combine_rows_per_src=[16374, 16384, 16380, 16380, 16384, 16358, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939098112 dispatch_meta_bytes=458816 combine_payload_bytes=939098112 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939556928 ingress_bytes=1819901952 link_bytes=1819901952 token_bytes=1878655040 weight_movement_bytes=880803840 total_nvlink_bytes=2759458880 weight_to_token_ratio=0.468848 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019069127e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119140 payload_rows_remote=102756 dup_rows=11932 dup_rows_remote=11916 dedup_ratio=0.091034 router_tokens_per_expert=[0, 8183, 0, 8192, 8, 0, 0, 7753, 0, 5251, 0, 0, 6, 3394, 0, 8192, 0, 8173, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16068, 14436, 6788, 16346, 16362, 16376, 16380] dup_rows_per_dst=[16, 5968, 5928, 12, 8, 0, 0, 0] combine_rows_per_src=[16384, 16068, 14436, 6788, 16346, 16362, 16376, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=458680 combine_payload_bytes=939163648 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939622328 ingress_bytes=1819967488 link_bytes=1819967488 token_bytes=1878785976 weight_movement_bytes=880803840 total_nvlink_bytes=2759589816 weight_to_token_ratio=0.468815 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019105824e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938196992 dispatch_meta_bytes=458752 combine_payload_bytes=938196992 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938655744 ingress_bytes=1290518528 link_bytes=1290518528 token_bytes=1876852736 weight_movement_bytes=352321536 total_nvlink_bytes=2229174272 weight_to_token_ratio=0.187719 gemm_rows_defined=65664 gemm_rows_charged=67584 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.028409 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226365068e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125264 payload_rows_remote=110920 dup_rows=5808 dup_rows_remote=2410 dedup_ratio=0.044312 router_tokens_per_expert=[5, 8186, 0, 8185, 0, 6, 0, 7590, 0, 6036, 0, 2, 0, 2787, 7, 8185, 9, 8131, 8181, 0, 6, 5, 8184, 9, 8, 1, 1, 3, 0, 3, 6, 0] payload_rows_per_dst=[16382, 15192, 13928, 14344, 16272, 16384, 16380, 16382] dup_rows_per_dst=[0, 0, 2364, 3398, 8, 24, 8, 6] combine_rows_per_src=[16382, 15192, 13928, 14344, 16272, 16384, 16380, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 2, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120708 payload_rows_remote=104516 dup_rows=10364 dup_rows_remote=202 dedup_ratio=0.079071 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 7861, 0, 5170, 12, 0, 0, 3442, 0, 8192, 0, 8091, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15722, 16192, 7076, 16182, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 10162, 202, 0, 0, 0, 0] combine_rows_per_src=[16384, 15722, 16192, 7076, 16182, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=458856 combine_payload_bytes=939376640 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939835496 ingress_bytes=1996341248 link_bytes=1996341248 token_bytes=1879212136 weight_movement_bytes=1056964608 total_nvlink_bytes=2936176744 weight_to_token_ratio=0.562451 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117867767e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=841777152 dispatch_meta_bytes=458688 combine_payload_bytes=841777152 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=842235840 ingress_bytes=1546420224 link_bytes=1546420224 token_bytes=1684012992 weight_movement_bytes=704643072 total_nvlink_bytes=2388656064 weight_to_token_ratio=0.418431 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.659307747e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=908656640 dispatch_meta_bytes=453320 combine_payload_bytes=908656640 prefetch_bytes=0 reduce_bytes=1585446912 egress_bytes=909109960 ingress_bytes=2494103552 link_bytes=2494103552 token_bytes=1817766600 weight_movement_bytes=1585446912 total_nvlink_bytes=3403213512 weight_to_token_ratio=0.872195 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=20608 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.396593880e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114648 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 8173, 0, 17, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 3, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16346, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 34, 0, 0, 0, 0, 6, 0] combine_rows_per_src=[16346, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123166 payload_rows_remote=106782 dup_rows=7906 dup_rows_remote=7906 dedup_ratio=0.060318 router_tokens_per_expert=[0, 8188, 0, 8192, 5, 0, 0, 7409, 0, 5053, 1, 0, 5, 3945, 0, 8192, 3, 8162, 8192, 1, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 14828, 10108, 16384, 16326, 16376, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 7900, 6, 0, 0, 0] combine_rows_per_src=[16376, 14828, 10108, 16384, 16326, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=856195072 dispatch_meta_bytes=418872 combine_payload_bytes=856195072 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=856613944 ingress_bytes=1208516608 link_bytes=1208516608 token_bytes=1712809016 weight_movement_bytes=352321536 total_nvlink_bytes=2065130552 weight_to_token_ratio=0.205698 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.767188545e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120858 payload_rows_remote=105096 dup_rows=10214 dup_rows_remote=10212 dedup_ratio=0.077927 router_tokens_per_expert=[0, 8190, 0, 8192, 6, 0, 0, 7876, 0, 5101, 0, 0, 5, 3402, 0, 8192, 0, 8190, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 15762, 16384, 6804, 16380, 16380, 16384, 16384] dup_rows_per_dst=[0, 2, 10202, 10, 0, 0, 0, 0] combine_rows_per_src=[16380, 15762, 16384, 6804, 16380, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125918 payload_rows_remote=109534 dup_rows=5154 dup_rows_remote=5154 dedup_ratio=0.039322 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 8108, 0, 5971, 0, 0, 0, 2366, 0, 8192, 0, 8131, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16216, 11942, 15960, 16264, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 4606, 538, 10, 0, 0] combine_rows_per_src=[16384, 16216, 11942, 15960, 16264, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458752 combine_payload_bytes=939196416 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939655168 ingress_bytes=1291517952 link_bytes=1291517952 token_bytes=1878851584 weight_movement_bytes=352321536 total_nvlink_bytes=2231173120 weight_to_token_ratio=0.187520 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231961425e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=874758144 dispatch_meta_bytes=458752 combine_payload_bytes=874758144 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=875216896 ingress_bytes=1227079680 link_bytes=1227079680 token_bytes=1749975040 weight_movement_bytes=352321536 total_nvlink_bytes=2102296576 weight_to_token_ratio=0.201329 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.871134000e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=860946432 dispatch_meta_bytes=461232 combine_payload_bytes=860946432 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=861407664 ingress_bytes=1213267968 link_bytes=1213267968 token_bytes=1722354096 weight_movement_bytes=352321536 total_nvlink_bytes=2074675632 weight_to_token_ratio=0.204558 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.793794178e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129906 payload_rows_remote=113532 dup_rows=1166 dup_rows_remote=1136 dedup_ratio=0.008896 router_tokens_per_expert=[0, 0, 0, 8187, 8192, 8192, 0, 0, 8192, 0, 0, 3, 0, 0, 72, 459, 8192, 0, 0, 8192, 8192, 475, 7186, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[16374, 16384, 16350, 16384, 16384, 15292, 16362, 16376] dup_rows_per_dst=[30, 68, 0, 1062, 2, 0, 0, 4] combine_rows_per_src=[16374, 16384, 16350, 16384, 16384, 15292, 16362, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124862 payload_rows_remote=108480 dup_rows=6210 dup_rows_remote=6208 dedup_ratio=0.047379 router_tokens_per_expert=[0, 8191, 0, 8191, 0, 1, 0, 7876, 1, 5400, 6, 1, 2, 3098, 1, 8190, 1, 8190, 8191, 0, 1, 1, 8190, 0, 1, 1, 0, 1, 0, 0, 1, 0] payload_rows_per_dst=[16382, 15754, 10814, 16382, 16380, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 6200, 2, 2, 4, 0] combine_rows_per_src=[16382, 15754, 10814, 16382, 16380, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129406 payload_rows_remote=113048 dup_rows=1666 dup_rows_remote=72 dedup_ratio=0.012711 router_tokens_per_expert=[9, 7, 4, 8185, 8185, 8174, 0, 7, 8185, 8, 7, 5, 0, 0, 86, 711, 8183, 0, 6, 8182, 8179, 355, 7028, 7, 0, 0, 8, 8, 0, 0, 0, 7] payload_rows_per_dst=[16384, 16362, 16384, 16358, 16376, 14778, 16384, 16380] dup_rows_per_dst=[26, 0, 26, 1594, 0, 2, 18, 0] combine_rows_per_src=[16384, 16362, 16384, 16358, 16376, 14778, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=897302528 dispatch_meta_bytes=458752 combine_payload_bytes=897302528 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=897761280 ingress_bytes=1249624064 link_bytes=1249624064 token_bytes=1795063808 weight_movement_bytes=352321536 total_nvlink_bytes=2147385344 weight_to_token_ratio=0.196272 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.997373140e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130344 payload_rows_remote=113962 dup_rows=728 dup_rows_remote=724 dedup_ratio=0.005554 router_tokens_per_expert=[0, 0, 0, 8188, 8192, 8190, 0, 0, 8191, 0, 0, 2, 0, 0, 36, 325, 8192, 0, 0, 8192, 8192, 455, 7380, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16380, 16382, 16384, 16384, 15670, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 722, 0, 0, 2, 0] combine_rows_per_src=[16376, 16380, 16382, 16384, 16384, 15670, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=930054144 dispatch_meta_bytes=458672 combine_payload_bytes=930054144 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=930512816 ingress_bytes=1634697216 link_bytes=1634697216 token_bytes=1860566960 weight_movement_bytes=704643072 total_nvlink_bytes=2565210032 weight_to_token_ratio=0.378725 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.153622053e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=888668160 dispatch_meta_bytes=458752 combine_payload_bytes=888668160 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=889126912 ingress_bytes=1417150464 link_bytes=1417150464 token_bytes=1777795072 weight_movement_bytes=528482304 total_nvlink_bytes=2306277376 weight_to_token_ratio=0.297268 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.935451051e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=926089216 dispatch_meta_bytes=452480 combine_payload_bytes=926089216 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=926541696 ingress_bytes=1278410752 link_bytes=1278410752 token_bytes=1852630912 weight_movement_bytes=352321536 total_nvlink_bytes=2204952448 weight_to_token_ratio=0.190174 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.158566576e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122450 payload_rows_remote=106090 dup_rows=8622 dup_rows_remote=8622 dedup_ratio=0.065781 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 7780, 0, 4278, 27, 0, 0, 4311, 0, 8192, 0, 8180, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15560, 8610, 16384, 16360, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 8622, 0, 0, 0, 0] combine_rows_per_src=[16384, 15560, 8610, 16384, 16360, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129924 payload_rows_remote=113540 dup_rows=1148 dup_rows_remote=1148 dedup_ratio=0.008759 router_tokens_per_expert=[0, 0, 0, 8186, 8192, 8192, 0, 0, 8192, 0, 0, 1, 0, 0, 7, 562, 8192, 0, 0, 8192, 8192, 346, 7278, 0, 0, 0, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[16372, 16384, 16384, 16384, 16384, 15248, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 1138, 0, 0, 0, 8] combine_rows_per_src=[16372, 16384, 16384, 16384, 16384, 15248, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130304 payload_rows_remote=113930 dup_rows=768 dup_rows_remote=768 dedup_ratio=0.005859 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 8187, 0, 0, 8191, 0, 0, 0, 0, 0, 91, 292, 8192, 0, 0, 8192, 8192, 272, 7542, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16374, 16382, 16384, 16384, 15628, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 766, 0, 0, 2, 0] combine_rows_per_src=[16384, 16374, 16382, 16384, 16384, 15628, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=933576704 dispatch_meta_bytes=458744 combine_payload_bytes=933576704 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=934035448 ingress_bytes=1814380544 link_bytes=1814380544 token_bytes=1867612152 weight_movement_bytes=880803840 total_nvlink_bytes=2748415992 weight_to_token_ratio=0.471620 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.015977369e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124160 payload_rows_remote=107776 dup_rows=6912 dup_rows_remote=6912 dedup_ratio=0.052734 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 32, 194, 8192, 0, 0, 8192, 8192, 110, 7857, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 16382, 16378, 9708, 16384, 16170, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 216, 6, 6686, 0, 0] combine_rows_per_src=[16370, 16382, 16378, 9708, 16384, 16170, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=869089280 dispatch_meta_bytes=458848 combine_payload_bytes=869089280 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=869548128 ingress_bytes=1573732352 link_bytes=1573732352 token_bytes=1738637408 weight_movement_bytes=704643072 total_nvlink_bytes=2443280480 weight_to_token_ratio=0.405285 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.812244263e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=930119680 dispatch_meta_bytes=458752 combine_payload_bytes=930119680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930578432 ingress_bytes=1282441216 link_bytes=1282441216 token_bytes=1860698112 weight_movement_bytes=352321536 total_nvlink_bytes=2213019648 weight_to_token_ratio=0.189349 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.181135492e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=933314560 dispatch_meta_bytes=458792 combine_payload_bytes=933314560 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=933773352 ingress_bytes=1637957632 link_bytes=1637957632 token_bytes=1867087912 weight_movement_bytes=704643072 total_nvlink_bytes=2571730984 weight_to_token_ratio=0.377402 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.171879022e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129650 payload_rows_remote=113266 dup_rows=1422 dup_rows_remote=126 dedup_ratio=0.010849 router_tokens_per_expert=[0, 0, 648, 0, 8192, 0, 0, 0, 8192, 8187, 8192, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 12, 0, 7690] payload_rows_per_dst=[16384, 16384, 16384, 16366, 16078, 16372, 16290, 15392] dup_rows_per_dst=[1296, 12, 102, 0, 0, 0, 0, 12] combine_rows_per_src=[16384, 16384, 16384, 16366, 16078, 16372, 16290, 15392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130370 payload_rows_remote=114648 dup_rows=702 dup_rows_remote=702 dedup_ratio=0.005356 router_tokens_per_expert=[1, 1, 1, 8186, 8191, 8177, 0, 1, 8189, 3, 1, 3, 0, 0, 92, 255, 8191, 0, 1, 8191, 8189, 140, 7720, 1, 0, 0, 0, 1, 0, 0, 0, 1] payload_rows_per_dst=[16374, 16356, 16384, 16382, 16384, 15722, 16384, 16384] dup_rows_per_dst=[4, 0, 8, 690, 0, 0, 0, 0] combine_rows_per_src=[16374, 16356, 16384, 16382, 16384, 15722, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=882900992 dispatch_meta_bytes=458752 combine_payload_bytes=882900992 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=883359744 ingress_bytes=1235222528 link_bytes=1235222528 token_bytes=1766260736 weight_movement_bytes=352321536 total_nvlink_bytes=2118582272 weight_to_token_ratio=0.199473 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.916730550e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129628 payload_rows_remote=113244 dup_rows=1444 dup_rows_remote=1428 dedup_ratio=0.011017 router_tokens_per_expert=[0, 0, 645, 7, 8179, 0, 6, 6, 8185, 8176, 8183, 7, 7, 8, 0, 0, 0, 8138, 6, 26, 8185, 5, 8, 3, 8181, 43, 0, 0, 5, 42, 0, 7485] payload_rows_per_dst=[16378, 16370, 16366, 16384, 16326, 16384, 16372, 15048] dup_rows_per_dst=[1292, 12, 0, 16, 14, 18, 76, 16] combine_rows_per_src=[16378, 16370, 16366, 16384, 16326, 16384, 16372, 15048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128656 payload_rows_remote=112280 dup_rows=2416 dup_rows_remote=2416 dedup_ratio=0.018433 router_tokens_per_expert=[4, 0, 1200, 0, 8192, 0, 0, 0, 8192, 8188, 8192, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 16, 0, 7022] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16292, 16384, 16384, 14068] dup_rows_per_dst=[2408, 0, 0, 0, 0, 0, 0, 8] combine_rows_per_src=[16384, 16384, 16376, 16384, 16292, 16384, 16384, 14068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=927875072 dispatch_meta_bytes=453568 combine_payload_bytes=927875072 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928328640 ingress_bytes=1280196608 link_bytes=1280196608 token_bytes=1856203712 weight_movement_bytes=352321536 total_nvlink_bytes=2208525248 weight_to_token_ratio=0.189808 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.168566624e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=461400 combine_payload_bytes=939196416 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939657816 ingress_bytes=1820000256 link_bytes=1820000256 token_bytes=1878854232 weight_movement_bytes=880803840 total_nvlink_bytes=2759658072 weight_to_token_ratio=0.468798 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019124173e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=927694848 dispatch_meta_bytes=458688 combine_payload_bytes=927694848 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928153536 ingress_bytes=1280016384 link_bytes=1280016384 token_bytes=1855848384 weight_movement_bytes=352321536 total_nvlink_bytes=2208169920 weight_to_token_ratio=0.189844 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.167557445e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 2, 2, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129402 payload_rows_remote=113018 dup_rows=1670 dup_rows_remote=1668 dedup_ratio=0.012741 router_tokens_per_expert=[0, 0, 802, 0, 8192, 0, 0, 0, 8192, 8187, 8192, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 75, 0, 7409] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16204, 16384, 16384, 14904] dup_rows_per_dst=[1604, 0, 0, 0, 0, 0, 2, 64] combine_rows_per_src=[16384, 16384, 16374, 16384, 16204, 16384, 16384, 14904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130556 payload_rows_remote=114172 dup_rows=516 dup_rows_remote=516 dedup_ratio=0.003937 router_tokens_per_expert=[0, 0, 0, 8189, 8192, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 9, 249, 8192, 0, 0, 8192, 8192, 62, 7876, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16382, 16384, 16384, 16384, 15876, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 516, 0, 0, 0, 0] combine_rows_per_src=[16378, 16382, 16384, 16384, 16384, 15876, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129408 payload_rows_remote=113024 dup_rows=1664 dup_rows_remote=1664 dedup_ratio=0.012695 router_tokens_per_expert=[0, 0, 814, 0, 8192, 0, 0, 0, 8192, 8186, 8192, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 25, 0, 7417] payload_rows_per_dst=[16384, 16384, 16372, 16384, 16270, 16384, 16382, 14848] dup_rows_per_dst=[1628, 0, 0, 0, 0, 0, 0, 36] combine_rows_per_src=[16384, 16384, 16372, 16384, 16270, 16384, 16382, 14848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=919797760 dispatch_meta_bytes=458784 combine_payload_bytes=919797760 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=920256544 ingress_bytes=2681405440 link_bytes=2681405440 token_bytes=1840054304 weight_movement_bytes=1761607680 total_nvlink_bytes=3601661984 weight_to_token_ratio=0.957367 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.501475119e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121042 payload_rows_remote=104696 dup_rows=10030 dup_rows_remote=1174 dedup_ratio=0.076523 router_tokens_per_expert=[0, 0, 66, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 154, 0, 8009] payload_rows_per_dst=[6542, 16384, 16316, 16384, 16310, 16376, 16384, 16346] dup_rows_per_dst=[54, 0, 20, 0, 1090, 0, 10, 8856] combine_rows_per_src=[6542, 16384, 16316, 16384, 16310, 16376, 16384, 16346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=925843456 dispatch_meta_bytes=458744 combine_payload_bytes=925843456 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=926302200 ingress_bytes=1454325760 link_bytes=1454325760 token_bytes=1852145656 weight_movement_bytes=528482304 total_nvlink_bytes=2380627960 weight_to_token_ratio=0.285335 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.143617191e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=935297024 dispatch_meta_bytes=458752 combine_payload_bytes=935297024 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=935755776 ingress_bytes=1639940096 link_bytes=1639940096 token_bytes=1871052800 weight_movement_bytes=704643072 total_nvlink_bytes=2575695872 weight_to_token_ratio=0.376602 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.182979993e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=925892608 dispatch_meta_bytes=458752 combine_payload_bytes=925892608 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=926351360 ingress_bytes=1102053376 link_bytes=1102053376 token_bytes=1852243968 weight_movement_bytes=176160768 total_nvlink_bytes=2028404736 weight_to_token_ratio=0.095107 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.171038886e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120808 payload_rows_remote=104424 dup_rows=10264 dup_rows_remote=10262 dedup_ratio=0.078308 router_tokens_per_expert=[0, 0, 8192, 0, 4, 0, 8192, 0, 0, 8186, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 11, 5051, 46, 8190, 0, 0, 0, 8192, 8189, 0, 5, 0, 3114] payload_rows_per_dst=[16384, 16384, 16372, 16336, 16384, 16384, 16326, 6238] dup_rows_per_dst=[2, 8, 20, 200, 22, 10012, 0, 0] combine_rows_per_src=[16384, 16384, 16372, 16336, 16384, 16384, 16326, 6238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129922 payload_rows_remote=113538 dup_rows=1150 dup_rows_remote=1148 dedup_ratio=0.008774 router_tokens_per_expert=[1, 1, 550, 0, 8185, 0, 1, 1, 8191, 8188, 8191, 1, 1, 1, 0, 0, 0, 8127, 0, 5, 8191, 1, 1, 0, 8190, 12, 0, 0, 0, 18, 1, 7678] payload_rows_per_dst=[16382, 16372, 16378, 16384, 16254, 16384, 16382, 15386] dup_rows_per_dst=[1104, 2, 0, 2, 10, 2, 22, 8] combine_rows_per_src=[16382, 16372, 16378, 16384, 16254, 16384, 16382, 15386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=857669632 dispatch_meta_bytes=423480 combine_payload_bytes=857669632 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=858093112 ingress_bytes=1209991168 link_bytes=1209991168 token_bytes=1715762744 weight_movement_bytes=352321536 total_nvlink_bytes=2068084280 weight_to_token_ratio=0.205344 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.775445466e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126042 payload_rows_remote=109668 dup_rows=5030 dup_rows_remote=5022 dedup_ratio=0.038376 router_tokens_per_expert=[4, 0, 8184, 9, 0, 2, 8183, 7, 0, 8185, 0, 7, 0, 4, 8, 8179, 8, 3, 5, 0, 5671, 0, 8111, 5, 7, 0, 8180, 8181, 3, 0, 47, 2543] payload_rows_per_dst=[16384, 16380, 16384, 16374, 16384, 11416, 16376, 16344] dup_rows_per_dst=[10, 4, 0, 8, 8, 32, 0, 4968] combine_rows_per_src=[16384, 16380, 16384, 16374, 16384, 11416, 16376, 16344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120772 payload_rows_remote=104394 dup_rows=10300 dup_rows_remote=10300 dedup_ratio=0.078583 router_tokens_per_expert=[0, 0, 8192, 0, 1, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 4, 6144, 12, 8187, 0, 0, 0, 8192, 8192, 0, 0, 0, 2051] payload_rows_per_dst=[16384, 16384, 16378, 16360, 16384, 15944, 16384, 6554] dup_rows_per_dst=[0, 2, 0, 0, 8, 9850, 0, 440] combine_rows_per_src=[16384, 16384, 16378, 16360, 16384, 15944, 16384, 6554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=855441408 dispatch_meta_bytes=458744 combine_payload_bytes=855441408 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=855900152 ingress_bytes=1207762944 link_bytes=1207762944 token_bytes=1711341560 weight_movement_bytes=352321536 total_nvlink_bytes=2063663096 weight_to_token_ratio=0.205874 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.762968341e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123224 payload_rows_remote=106842 dup_rows=7848 dup_rows_remote=7848 dedup_ratio=0.059875 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8191, 0, 0, 8185, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 8, 4281, 28, 8185, 0, 0, 0, 8192, 8191, 0, 2, 0, 3915] payload_rows_per_dst=[16384, 16382, 16370, 16332, 16384, 8606, 16382, 16384] dup_rows_per_dst=[0, 0, 0, 0, 16, 12, 0, 7820] combine_rows_per_src=[16384, 16382, 16370, 16332, 16384, 8606, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120290 payload_rows_remote=103906 dup_rows=10782 dup_rows_remote=10780 dedup_ratio=0.082260 router_tokens_per_expert=[0, 0, 8192, 0, 1, 0, 8192, 0, 0, 8186, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 7, 5396, 19, 8160, 0, 0, 0, 8192, 8192, 0, 0, 0, 2827] payload_rows_per_dst=[16384, 16384, 16372, 16344, 16384, 16384, 16384, 5654] dup_rows_per_dst=[0, 2, 0, 0, 14, 10766, 0, 0] combine_rows_per_src=[16384, 16384, 16372, 16344, 16384, 16384, 16384, 5654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=930103296 dispatch_meta_bytes=458744 combine_payload_bytes=930103296 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=930562040 ingress_bytes=1282424832 link_bytes=1282424832 token_bytes=1860665336 weight_movement_bytes=352321536 total_nvlink_bytes=2212986872 weight_to_token_ratio=0.189352 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.181043749e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=898400256 dispatch_meta_bytes=458760 combine_payload_bytes=898400256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=898859016 ingress_bytes=1250721792 link_bytes=1250721792 token_bytes=1797259272 weight_movement_bytes=352321536 total_nvlink_bytes=2149580808 weight_to_token_ratio=0.196033 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.003519958e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130542 payload_rows_remote=114318 dup_rows=530 dup_rows_remote=530 dedup_ratio=0.004044 router_tokens_per_expert=[0, 0, 232, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 36, 0, 8004] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16224, 16384, 16384, 16014] dup_rows_per_dst=[464, 0, 0, 0, 0, 0, 0, 66] combine_rows_per_src=[16384, 16384, 16384, 16384, 16224, 16384, 16384, 16014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=855195648 dispatch_meta_bytes=458776 combine_payload_bytes=855195648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=855654424 ingress_bytes=1207517184 link_bytes=1207517184 token_bytes=1710850072 weight_movement_bytes=352321536 total_nvlink_bytes=2063171608 weight_to_token_ratio=0.205934 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.761592188e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=875249664 dispatch_meta_bytes=458760 combine_payload_bytes=875249664 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=875708424 ingress_bytes=1932214272 link_bytes=1932214272 token_bytes=1750958088 weight_movement_bytes=1056964608 total_nvlink_bytes=2807922696 weight_to_token_ratio=0.603649 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.081959338e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121292 payload_rows_remote=104908 dup_rows=9780 dup_rows_remote=98 dedup_ratio=0.074615 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 3353, 4, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 4841] payload_rows_per_dst=[16372, 16370, 16384, 16372, 16312, 6714, 16384, 16384] dup_rows_per_dst=[0, 0, 26, 0, 0, 0, 72, 9682] combine_rows_per_src=[16372, 16370, 16384, 16372, 16312, 6714, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=851197952 dispatch_meta_bytes=458744 combine_payload_bytes=851197952 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=851656696 ingress_bytes=1203519488 link_bytes=1203519488 token_bytes=1702854648 weight_movement_bytes=352321536 total_nvlink_bytes=2055176184 weight_to_token_ratio=0.206901 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739206759e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936493056 dispatch_meta_bytes=459392 combine_payload_bytes=936493056 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936952448 ingress_bytes=1288814592 link_bytes=1288814592 token_bytes=1873445504 weight_movement_bytes=352321536 total_nvlink_bytes=2225767040 weight_to_token_ratio=0.188061 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.216823738e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131026 payload_rows_remote=114644 dup_rows=46 dup_rows_remote=24 dedup_ratio=0.000351 router_tokens_per_expert=[0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 1, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16382, 16374, 16380, 16378, 16384, 16362] dup_rows_per_dst=[22, 10, 0, 0, 0, 2, 10, 2] combine_rows_per_src=[16382, 16384, 16382, 16374, 16380, 16378, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=859406336 dispatch_meta_bytes=420024 combine_payload_bytes=859406336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=859826360 ingress_bytes=1211727872 link_bytes=1211727872 token_bytes=1719232696 weight_movement_bytes=352321536 total_nvlink_bytes=2071554232 weight_to_token_ratio=0.204930 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=21888 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.785170283e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128222 payload_rows_remote=114656 dup_rows=2850 dup_rows_remote=2848 dedup_ratio=0.021744 router_tokens_per_expert=[0, 0, 8191, 1, 3, 0, 8190, 1, 0, 8190, 0, 1, 1, 1, 1, 8181, 0, 0, 1, 2, 6773, 10, 8177, 1, 0, 0, 8191, 8190, 0, 1, 6, 1423] payload_rows_per_dst=[16384, 16384, 16382, 16364, 16384, 13566, 16380, 16378] dup_rows_per_dst=[0, 4, 0, 4, 4, 2, 0, 2836] combine_rows_per_src=[16384, 16384, 16382, 16364, 16384, 13566, 16380, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130980 payload_rows_remote=114596 dup_rows=92 dup_rows_remote=82 dedup_ratio=0.000702 router_tokens_per_expert=[7, 7, 8174, 8170, 4, 0, 0, 0, 0, 3, 8, 8, 0, 5, 10, 8182, 8179, 11, 3, 8183, 0, 8180, 15, 0, 8185, 1, 8182, 3, 0, 0, 4, 12] payload_rows_per_dst=[16360, 16374, 16364, 16384, 16380, 16374, 16370, 16374] dup_rows_per_dst=[8, 0, 22, 10, 6, 16, 2, 28] combine_rows_per_src=[16360, 16374, 16364, 16384, 16380, 16374, 16370, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=458672 combine_payload_bytes=939163648 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939622320 ingress_bytes=1467645952 link_bytes=1467645952 token_bytes=1878785968 weight_movement_bytes=528482304 total_nvlink_bytes=2407268272 weight_to_token_ratio=0.281289 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218204706e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126066 payload_rows_remote=109682 dup_rows=5006 dup_rows_remote=5006 dedup_ratio=0.038193 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 5680, 12, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 2503] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16384, 11384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 5006] combine_rows_per_src=[16384, 16384, 16384, 16378, 16384, 11384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=470016 combine_payload_bytes=939261952 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939731968 ingress_bytes=1996226560 link_bytes=1996226560 token_bytes=1878993920 weight_movement_bytes=1056964608 total_nvlink_bytes=2935958528 weight_to_token_ratio=0.562516 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=21888 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117803547e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938770432 dispatch_meta_bytes=458712 combine_payload_bytes=938770432 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939229144 ingress_bytes=1291091968 link_bytes=1291091968 token_bytes=1877999576 weight_movement_bytes=352321536 total_nvlink_bytes=2230321112 weight_to_token_ratio=0.187605 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229576092e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879506944 weight_movement_bytes=352321536 total_nvlink_bytes=2231828480 weight_to_token_ratio=0.187454 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=939982848 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879506944 weight_movement_bytes=704643072 total_nvlink_bytes=2584150016 weight_to_token_ratio=0.374908 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879506944 weight_movement_bytes=528482304 total_nvlink_bytes=2407989248 weight_to_token_ratio=0.281181 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=898514944 dispatch_meta_bytes=458752 combine_payload_bytes=898514944 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=898973696 ingress_bytes=1250836480 link_bytes=1250836480 token_bytes=1797488640 weight_movement_bytes=352321536 total_nvlink_bytes=2149810176 weight_to_token_ratio=0.196008 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.004162163e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116422 payload_rows_remote=100038 dup_rows=14650 dup_rows_remote=584 dedup_ratio=0.111771 router_tokens_per_expert=[7021, 8192, 0, 0, 0, 8192, 282, 0, 0, 0, 8192, 8192, 8192, 0, 0, 889, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 1778, 16360, 16384, 16364, 16384] dup_rows_per_dst=[14066, 564, 20, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 1778, 16360, 16384, 16364, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879506944 weight_movement_bytes=352321536 total_nvlink_bytes=2231828480 weight_to_token_ratio=0.187454 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114680 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[1, 1, 8190, 8188, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 8191, 8191, 1, 1, 8191, 0, 8190, 1, 0, 8191, 1, 8191, 1, 0, 0, 0, 2] payload_rows_per_dst=[16380, 16384, 16382, 16384, 16384, 16382, 16384, 16382] dup_rows_per_dst=[0, 0, 2, 0, 2, 0, 2, 4] combine_rows_per_src=[16380, 16384, 16382, 16384, 16384, 16382, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=819511296 dispatch_meta_bytes=402488 combine_payload_bytes=819511296 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=819913784 ingress_bytes=1524154368 link_bytes=1524154368 token_bytes=1639425080 weight_movement_bytes=704643072 total_nvlink_bytes=2344068152 weight_to_token_ratio=0.429811 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.534628247e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127432 payload_rows_remote=111048 dup_rows=3640 dup_rows_remote=2162 dedup_ratio=0.027771 router_tokens_per_expert=[6396, 8153, 9, 0, 0, 8015, 1214, 0, 6, 5, 8183, 8178, 8182, 12, 0, 737, 7, 1, 9, 35, 4, 0, 8180, 8, 0, 9, 9, 0, 0, 1, 0, 8183] payload_rows_per_dst=[12810, 16356, 16370, 16384, 16384, 16376, 16384, 16368] dup_rows_per_dst=[0, 2102, 8, 1478, 26, 8, 18, 0] combine_rows_per_src=[12810, 16356, 16370, 16384, 16384, 16376, 16384, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126910 payload_rows_remote=110526 dup_rows=4162 dup_rows_remote=4162 dedup_ratio=0.031754 router_tokens_per_expert=[6111, 8192, 0, 0, 0, 8192, 1343, 0, 0, 0, 8192, 8192, 8192, 0, 0, 738, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[12222, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 2686, 0, 1476, 0, 0, 0, 0] combine_rows_per_src=[12222, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128070 payload_rows_remote=111686 dup_rows=3002 dup_rows_remote=3002 dedup_ratio=0.022903 router_tokens_per_expert=[6691, 8192, 0, 0, 0, 8192, 686, 5, 0, 0, 8192, 8192, 8192, 0, 0, 810, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[13382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 1382, 0, 1620, 0, 0, 0, 0] combine_rows_per_src=[13382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458760 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939917320 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879375880 weight_movement_bytes=352321536 total_nvlink_bytes=2231697416 weight_to_token_ratio=0.187467 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120186 payload_rows_remote=103802 dup_rows=10886 dup_rows_remote=9562 dedup_ratio=0.083054 router_tokens_per_expert=[6162, 8192, 0, 0, 0, 8192, 662, 0, 0, 0, 8192, 8192, 8192, 0, 0, 1368, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[14240, 16384, 16384, 7642, 16384, 16384, 16384, 16384] dup_rows_per_dst=[7860, 1324, 0, 1702, 0, 0, 0, 0] combine_rows_per_src=[14240, 16384, 16384, 7642, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=909705216 dispatch_meta_bytes=452840 combine_payload_bytes=909705216 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=910158056 ingress_bytes=1614348288 link_bytes=1614348288 token_bytes=1819863272 weight_movement_bytes=704643072 total_nvlink_bytes=2524506344 weight_to_token_ratio=0.387196 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.039676551e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=905428992 dispatch_meta_bytes=458752 combine_payload_bytes=905428992 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=905887744 ingress_bytes=1610072064 link_bytes=1610072064 token_bytes=1811316736 weight_movement_bytes=704643072 total_nvlink_bytes=2515959808 weight_to_token_ratio=0.389023 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.015731481e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118004 payload_rows_remote=101620 dup_rows=13068 dup_rows_remote=13052 dedup_ratio=0.099701 router_tokens_per_expert=[7118, 8192, 0, 0, 0, 8192, 416, 0, 0, 0, 8192, 8192, 8192, 0, 0, 658, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[14236, 5480, 16368, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 170, 4, 12878, 0, 0, 0, 16] combine_rows_per_src=[14236, 5480, 16368, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 3, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=914931712 dispatch_meta_bytes=458752 combine_payload_bytes=914931712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=915390464 ingress_bytes=1267253248 link_bytes=1267253248 token_bytes=1830322176 weight_movement_bytes=352321536 total_nvlink_bytes=2182643712 weight_to_token_ratio=0.192492 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.096089211e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=850345984 dispatch_meta_bytes=453456 combine_payload_bytes=850345984 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=850799440 ingress_bytes=1378828288 link_bytes=1378828288 token_bytes=1701145424 weight_movement_bytes=528482304 total_nvlink_bytes=2229627728 weight_to_token_ratio=0.310663 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=30080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.720862862e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129298 payload_rows_remote=112914 dup_rows=1774 dup_rows_remote=1756 dedup_ratio=0.013535 router_tokens_per_expert=[0, 0, 0, 0, 0, 8143, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8186, 0, 0, 8191, 0, 50, 8192, 8192, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 15630, 16098, 15784, 16250] dup_rows_per_dst=[18, 958, 0, 744, 0, 54, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 15630, 16098, 15784, 16250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939982848 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879506944 weight_movement_bytes=704643072 total_nvlink_bytes=2584150016 weight_to_token_ratio=0.374908 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 6, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=832471040 dispatch_meta_bytes=458688 combine_payload_bytes=832471040 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=832929728 ingress_bytes=1008631808 link_bytes=1008631808 token_bytes=1665400768 weight_movement_bytes=176160768 total_nvlink_bytes=1841561536 weight_to_token_ratio=0.105777 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.647917101e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127320 payload_rows_remote=110938 dup_rows=3752 dup_rows_remote=3750 dedup_ratio=0.028625 router_tokens_per_expert=[6321, 8181, 1, 0, 0, 8161, 570, 1, 0, 1, 8191, 8190, 8191, 2, 0, 1330, 1, 0, 1, 8, 1, 0, 8190, 1, 0, 1, 1, 0, 0, 1, 0, 8191] payload_rows_per_dst=[12644, 16380, 16382, 16384, 16380, 16382, 16384, 16384] dup_rows_per_dst=[0, 1084, 0, 2662, 2, 2, 2, 0] combine_rows_per_src=[12644, 16380, 16382, 16384, 16380, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=924991488 dispatch_meta_bytes=458680 combine_payload_bytes=924991488 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=925450168 ingress_bytes=1453473792 link_bytes=1453473792 token_bytes=1850441656 weight_movement_bytes=528482304 total_nvlink_bytes=2378923960 weight_to_token_ratio=0.285598 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.138846525e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129862 payload_rows_remote=114064 dup_rows=1210 dup_rows_remote=1210 dedup_ratio=0.009232 router_tokens_per_expert=[12, 6, 19, 9, 52, 7936, 0, 8182, 0, 12, 8183, 9, 10, 0, 0, 7889, 330, 8163, 0, 0, 8135, 0, 1, 8135, 8179, 4, 4, 213, 42, 0, 9, 2] payload_rows_per_dst=[16308, 15976, 16382, 15798, 16362, 16270, 16384, 16382] dup_rows_per_dst=[54, 0, 26, 0, 624, 2, 416, 88] combine_rows_per_src=[16308, 15976, 16382, 15798, 16362, 16270, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130988 payload_rows_remote=114604 dup_rows=84 dup_rows_remote=84 dedup_ratio=0.000641 router_tokens_per_expert=[0, 0, 0, 0, 0, 8152, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 8191, 0, 42, 8192, 8192, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16304, 16384, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 82, 0, 2] combine_rows_per_src=[16384, 16304, 16384, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130936 payload_rows_remote=114552 dup_rows=136 dup_rows_remote=136 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 0, 8125, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8188, 0, 67, 8192, 8192, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[16382, 16250, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 128, 0, 8] combine_rows_per_src=[16382, 16250, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=908804096 dispatch_meta_bytes=458752 combine_payload_bytes=908804096 prefetch_bytes=0 reduce_bytes=0 egress_bytes=909262848 ingress_bytes=908804096 link_bytes=909262848 token_bytes=1818066944 weight_movement_bytes=0 total_nvlink_bytes=1818066944 weight_to_token_ratio=0.000000 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.091492404e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130988 payload_rows_remote=114678 dup_rows=84 dup_rows_remote=84 dedup_ratio=0.000641 router_tokens_per_expert=[0, 0, 0, 0, 0, 8155, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 8188, 0, 0, 8192, 0, 38, 8192, 8192, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[16384, 16310, 16384, 16382, 16376, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 76, 0, 8] combine_rows_per_src=[16384, 16310, 16384, 16382, 16376, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934412288 dispatch_meta_bytes=461096 combine_payload_bytes=934412288 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934873384 ingress_bytes=1286733824 link_bytes=1286733824 token_bytes=1869285672 weight_movement_bytes=352321536 total_nvlink_bytes=2221607208 weight_to_token_ratio=0.188479 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.205172305e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127916 payload_rows_remote=111532 dup_rows=3156 dup_rows_remote=3156 dedup_ratio=0.024078 router_tokens_per_expert=[6614, 8192, 0, 0, 0, 8192, 812, 0, 0, 0, 8192, 8192, 8192, 0, 0, 766, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[13228, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 1624, 0, 1532, 0, 0, 0, 0] combine_rows_per_src=[13228, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938835968 dispatch_meta_bytes=458752 combine_payload_bytes=938835968 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=939294720 ingress_bytes=1291157504 link_bytes=1291157504 token_bytes=1878130688 weight_movement_bytes=352321536 total_nvlink_bytes=2230452224 weight_to_token_ratio=0.187592 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229943067e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130922 payload_rows_remote=114538 dup_rows=150 dup_rows_remote=150 dedup_ratio=0.001144 router_tokens_per_expert=[0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 12, 8192, 8192, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16360, 16360, 16384, 16384, 16282, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 12, 24, 114, 0] combine_rows_per_src=[16384, 16360, 16360, 16384, 16384, 16282, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938409984 dispatch_meta_bytes=458752 combine_payload_bytes=938409984 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938868736 ingress_bytes=1290731520 link_bytes=1290731520 token_bytes=1877278720 weight_movement_bytes=352321536 total_nvlink_bytes=2229600256 weight_to_token_ratio=0.187677 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227557734e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131040 payload_rows_remote=114656 dup_rows=32 dup_rows_remote=28 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 14, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8176, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16352] dup_rows_per_dst=[4, 0, 0, 28, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=459048 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939901224 ingress_bytes=1644085248 link_bytes=1644085248 token_bytes=1879343400 weight_movement_bytes=704643072 total_nvlink_bytes=2583986472 weight_to_token_ratio=0.374941 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206191114e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=913670144 dispatch_meta_bytes=458752 combine_payload_bytes=913670144 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=914128896 ingress_bytes=1265991680 link_bytes=1265991680 token_bytes=1827799040 weight_movement_bytes=352321536 total_nvlink_bytes=2180120576 weight_to_token_ratio=0.192757 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.089024957e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938295296 dispatch_meta_bytes=458752 combine_payload_bytes=938295296 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938754048 ingress_bytes=1290616832 link_bytes=1290616832 token_bytes=1877049344 weight_movement_bytes=352321536 total_nvlink_bytes=2229370880 weight_to_token_ratio=0.187700 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226915529e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=458736 combine_payload_bytes=939261952 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939720688 ingress_bytes=1291583488 link_bytes=1291583488 token_bytes=1878982640 weight_movement_bytes=352321536 total_nvlink_bytes=2231304176 weight_to_token_ratio=0.187507 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232328399e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130750 payload_rows_remote=114394 dup_rows=322 dup_rows_remote=260 dedup_ratio=0.002457 router_tokens_per_expert=[2, 0, 3, 1, 1, 8123, 0, 8191, 0, 1, 8191, 0, 1, 1, 0, 8114, 77, 8189, 0, 0, 8188, 0, 32, 8178, 8190, 0, 1, 36, 13, 1, 1, 1] payload_rows_per_dst=[16384, 16248, 16382, 16230, 16382, 16356, 16384, 16384] dup_rows_per_dst=[6, 0, 2, 2, 150, 62, 70, 30] combine_rows_per_src=[16384, 16248, 16382, 16230, 16382, 16356, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130858 payload_rows_remote=114524 dup_rows=214 dup_rows_remote=206 dedup_ratio=0.001633 router_tokens_per_expert=[1, 0, 0, 8, 10, 0, 56, 11, 29, 8, 8181, 8180, 4, 0, 0, 10, 8130, 0, 8164, 11, 8, 8173, 0, 21, 0, 1, 7, 8176, 0, 8, 8182, 8157] payload_rows_per_dst=[16378, 16384, 16368, 16334, 16282, 16382, 16366, 16364] dup_rows_per_dst=[0, 98, 68, 8, 0, 22, 2, 16] combine_rows_per_src=[16378, 16384, 16368, 16334, 16282, 16382, 16366, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[3, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131054 payload_rows_remote=114670 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000137 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 9, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8183, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16366] dup_rows_per_dst=[0, 0, 0, 18, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=937115648 dispatch_meta_bytes=458616 combine_payload_bytes=937115648 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=937574264 ingress_bytes=2170241024 link_bytes=2170241024 token_bytes=1874689912 weight_movement_bytes=1233125376 total_nvlink_bytes=3107815288 weight_to_token_ratio=0.657776 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.215244383e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131052 payload_rows_remote=114668 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 10, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8182, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16364] dup_rows_per_dst=[0, 0, 0, 20, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938180608 dispatch_meta_bytes=458920 combine_payload_bytes=938180608 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938639528 ingress_bytes=1290502144 link_bytes=1290502144 token_bytes=1876820136 weight_movement_bytes=352321536 total_nvlink_bytes=2229141672 weight_to_token_ratio=0.187723 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226273324e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=939966464 ingress_bytes=1820311552 link_bytes=1820311552 token_bytes=1879474176 weight_movement_bytes=880803840 total_nvlink_bytes=2760278016 weight_to_token_ratio=0.468644 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019298486e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130970 payload_rows_remote=114586 dup_rows=102 dup_rows_remote=102 dedup_ratio=0.000778 router_tokens_per_expert=[0, 0, 0, 0, 0, 8141, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 51, 8192, 8192, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16282, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 102, 0, 0] combine_rows_per_src=[16384, 16282, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130854 payload_rows_remote=114472 dup_rows=218 dup_rows_remote=136 dedup_ratio=0.001663 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 8192] payload_rows_per_dst=[16384, 16278, 16324, 16352, 16384, 16366, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 2, 114, 2, 18, 82] combine_rows_per_src=[16384, 16278, 16324, 16352, 16384, 16366, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=458752 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939835392 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879212032 weight_movement_bytes=176160768 total_nvlink_bytes=2055372800 weight_to_token_ratio=0.093742 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130820 payload_rows_remote=114436 dup_rows=252 dup_rows_remote=228 dedup_ratio=0.001923 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8187, 0, 0, 8192, 0, 0, 0, 40, 0, 1, 0, 0, 8151, 0, 8192, 8192, 8192, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[16384, 16384, 16348, 16360, 16384, 16378, 16244, 16338] dup_rows_per_dst=[24, 26, 6, 0, 82, 114, 0, 0] combine_rows_per_src=[16384, 16384, 16348, 16360, 16384, 16378, 16244, 16338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458752 combine_payload_bytes=939360256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939819008 ingress_bytes=1291681792 link_bytes=1291681792 token_bytes=1879179264 weight_movement_bytes=352321536 total_nvlink_bytes=2231500800 weight_to_token_ratio=0.187487 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232878861e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938688512 dispatch_meta_bytes=458752 combine_payload_bytes=938688512 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939147264 ingress_bytes=1291010048 link_bytes=1291010048 token_bytes=1877835776 weight_movement_bytes=352321536 total_nvlink_bytes=2230157312 weight_to_token_ratio=0.187621 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229117375e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=937754624 dispatch_meta_bytes=458432 combine_payload_bytes=937754624 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938213056 ingress_bytes=1466236928 link_bytes=1466236928 token_bytes=1875967680 weight_movement_bytes=528482304 total_nvlink_bytes=2404449984 weight_to_token_ratio=0.281712 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.210314759e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114660 dup_rows=34 dup_rows_remote=32 dedup_ratio=0.000259 router_tokens_per_expert=[1, 0, 0, 1, 1, 0, 8, 1, 6, 1, 8189, 8191, 1, 0, 3, 1, 8185, 0, 8186, 1, 1, 8187, 0, 2, 0, 0, 1, 8190, 0, 2, 8188, 8189] payload_rows_per_dst=[16384, 16384, 16382, 16380, 16372, 16378, 16382, 16376] dup_rows_per_dst=[2, 8, 10, 8, 0, 2, 0, 4] combine_rows_per_src=[16384, 16384, 16382, 16380, 16372, 16378, 16382, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937459712 dispatch_meta_bytes=458656 combine_payload_bytes=937459712 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=937918368 ingress_bytes=1465942016 link_bytes=1465942016 token_bytes=1875378080 weight_movement_bytes=528482304 total_nvlink_bytes=2403860384 weight_to_token_ratio=0.281800 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.208663375e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130820 payload_rows_remote=114440 dup_rows=252 dup_rows_remote=230 dedup_ratio=0.001923 router_tokens_per_expert=[12, 0, 9, 8183, 0, 10, 0, 8170, 7, 8181, 11, 11, 8181, 15, 0, 5, 0, 0, 91, 0, 11, 8176, 0, 8179, 8093, 8168, 0, 3, 9, 0, 0, 11] payload_rows_per_dst=[16384, 16360, 16384, 16380, 16366, 16374, 16192, 16380] dup_rows_per_dst=[24, 0, 36, 22, 152, 0, 0, 18] combine_rows_per_src=[16384, 16360, 16384, 16380, 16366, 16374, 16192, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131022 payload_rows_remote=114638 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000381 router_tokens_per_expert=[0, 1, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 24, 0, 0, 0, 0, 8167, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16334, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 0, 48, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16334, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131004 payload_rows_remote=114620 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.000519 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 8191, 0, 0, 0, 21, 0, 0, 0, 0, 8162, 0, 8192, 8192, 8192, 0, 0, 0, 12, 0, 1] payload_rows_per_dst=[16384, 16384, 16378, 16382, 16384, 16324, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 42, 0, 0, 26] combine_rows_per_src=[16384, 16384, 16378, 16382, 16384, 16324, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131026 payload_rows_remote=114642 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000351 router_tokens_per_expert=[0, 1, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 22, 0, 0, 0, 0, 8169, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16338, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 0, 44, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16338, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=458768 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939753488 ingress_bytes=1291616256 link_bytes=1291616256 token_bytes=1879048208 weight_movement_bytes=352321536 total_nvlink_bytes=2231369744 weight_to_token_ratio=0.187500 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232511886e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937492480 dispatch_meta_bytes=458680 combine_payload_bytes=937492480 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=937951160 ingress_bytes=1465974784 link_bytes=1465974784 token_bytes=1875443640 weight_movement_bytes=528482304 total_nvlink_bytes=2403925944 weight_to_token_ratio=0.281791 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.208846862e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=458752 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939573248 ingress_bytes=1467596800 link_bytes=1467596800 token_bytes=1878687744 weight_movement_bytes=528482304 total_nvlink_bytes=2407170048 weight_to_token_ratio=0.281304 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217929475e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130578 payload_rows_remote=114194 dup_rows=494 dup_rows_remote=494 dedup_ratio=0.003769 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 10, 0, 0, 0, 0, 8182, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16330, 16384, 16382, 16384, 15966, 16364, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 4, 20, 122, 348, 0] combine_rows_per_src=[16330, 16384, 16382, 16384, 15966, 16364, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938967040 dispatch_meta_bytes=458752 combine_payload_bytes=938967040 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=939425792 ingress_bytes=1995931648 link_bytes=1995931648 token_bytes=1878392832 weight_movement_bytes=1056964608 total_nvlink_bytes=2935357440 weight_to_token_ratio=0.562696 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117638409e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=458752 combine_payload_bytes=939147264 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939606016 ingress_bytes=1115308032 link_bytes=1115308032 token_bytes=1878753280 weight_movement_bytes=176160768 total_nvlink_bytes=2054914048 weight_to_token_ratio=0.093765 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245259427e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130620 payload_rows_remote=114236 dup_rows=452 dup_rows_remote=76 dedup_ratio=0.003448 router_tokens_per_expert=[0, 0, 8186, 8192, 0, 0, 8181, 0, 0, 0, 8192, 142, 0, 8192, 0, 0, 0, 0, 0, 0, 11, 0, 6, 0, 0, 0, 8192, 0, 0, 8050, 0, 8192] payload_rows_per_dst=[16384, 16362, 16250, 16384, 16372, 16384, 16384, 16100] dup_rows_per_dst=[376, 14, 28, 0, 0, 34, 0, 0] combine_rows_per_src=[16384, 16362, 16250, 16384, 16372, 16384, 16384, 16100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939982848 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879506944 weight_movement_bytes=528482304 total_nvlink_bytes=2407989248 weight_to_token_ratio=0.281181 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=935477248 dispatch_meta_bytes=458752 combine_payload_bytes=935477248 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935936000 ingress_bytes=1287798784 link_bytes=1287798784 token_bytes=1871413248 weight_movement_bytes=352321536 total_nvlink_bytes=2223734784 weight_to_token_ratio=0.188265 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.211135637e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114664 dup_rows=34 dup_rows_remote=32 dedup_ratio=0.000259 router_tokens_per_expert=[1, 1, 1, 8190, 0, 1, 0, 8190, 1, 8190, 1, 1, 8191, 2, 0, 1, 2, 0, 7, 1, 1, 8186, 1, 8190, 8184, 8188, 0, 0, 1, 4, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16382, 16374, 16368, 16382] dup_rows_per_dst=[2, 0, 4, 4, 14, 2, 0, 8] combine_rows_per_src=[16384, 16382, 16382, 16384, 16382, 16374, 16368, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=935821312 dispatch_meta_bytes=457248 combine_payload_bytes=935821312 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=936278560 ingress_bytes=1816625152 link_bytes=1816625152 token_bytes=1872099872 weight_movement_bytes=880803840 total_nvlink_bytes=2752903712 weight_to_token_ratio=0.470490 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.017234255e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128054 payload_rows_remote=111692 dup_rows=3018 dup_rows_remote=3012 dedup_ratio=0.023026 router_tokens_per_expert=[9, 1422, 8180, 8128, 14, 16, 8180, 63, 7, 0, 8071, 49, 10, 8169, 0, 5, 0, 8, 12, 0, 0, 14, 0, 0, 0, 0, 8182, 9, 0, 6812, 0, 8176] payload_rows_per_dst=[16366, 16384, 16172, 16362, 16384, 16380, 16382, 13624] dup_rows_per_dst=[2752, 162, 82, 6, 16, 0, 0, 0] combine_rows_per_src=[16366, 16384, 16172, 16362, 16384, 16380, 16382, 13624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130986 payload_rows_remote=114602 dup_rows=86 dup_rows_remote=10 dedup_ratio=0.000656 router_tokens_per_expert=[0, 2, 8192, 8192, 0, 0, 8189, 0, 0, 0, 8192, 38, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 8152, 0, 8192] payload_rows_per_dst=[16384, 16378, 16384, 16384, 16384, 16384, 16384, 16304] dup_rows_per_dst=[4, 0, 76, 0, 0, 6, 0, 0] combine_rows_per_src=[16384, 16378, 16384, 16384, 16384, 16384, 16384, 16304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130890 payload_rows_remote=114506 dup_rows=182 dup_rows_remote=182 dedup_ratio=0.001389 router_tokens_per_expert=[0, 0, 8188, 8192, 0, 0, 8188, 0, 0, 0, 8192, 83, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 8192, 0, 0, 8105, 4, 8192] payload_rows_per_dst=[16378, 16376, 16384, 16384, 16382, 16384, 16384, 16218] dup_rows_per_dst=[0, 0, 166, 0, 0, 16, 0, 0] combine_rows_per_src=[16378, 16376, 16384, 16384, 16382, 16384, 16384, 16218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130954 payload_rows_remote=114582 dup_rows=118 dup_rows_remote=118 dedup_ratio=0.000900 router_tokens_per_expert=[0, 2, 8190, 8192, 0, 0, 8186, 0, 0, 0, 8192, 49, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 0, 0, 8192, 0, 0, 8126, 15, 8192] payload_rows_per_dst=[16384, 16372, 16384, 16384, 16380, 16384, 16384, 16282] dup_rows_per_dst=[4, 0, 98, 0, 0, 16, 0, 0] combine_rows_per_src=[16384, 16372, 16384, 16384, 16380, 16384, 16384, 16282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=458784 combine_payload_bytes=939327488 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939786272 ingress_bytes=1643970560 link_bytes=1643970560 token_bytes=1879113760 weight_movement_bytes=704643072 total_nvlink_bytes=2583756832 weight_to_token_ratio=0.374987 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205548909e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28421566416 prefetch_bytes=5108662272 reduce_bytes=4580179968 weight_movement_bytes=9688842240 egress_bytes=14214410192 ingress_bytes=23895998464 link_bytes=23895998464 total_nvlink_bytes=38110408656 weight_to_token_ratio=0.340898 prefetch_share_of_ingress=0.213787 dedup_ratio=0.048739 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=26 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.021188 gemm_inner_pad_upper_bound=18542 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.362976045e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938819584 dispatch_meta_bytes=458448 combine_payload_bytes=938819584 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939278032 ingress_bytes=1819623424 link_bytes=1819623424 token_bytes=1878097616 weight_movement_bytes=880803840 total_nvlink_bytes=2758901456 weight_to_token_ratio=0.468987 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018913163e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=914980864 dispatch_meta_bytes=458816 combine_payload_bytes=914980864 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=915439680 ingress_bytes=1091141632 link_bytes=1091141632 token_bytes=1830420544 weight_movement_bytes=176160768 total_nvlink_bytes=2006581312 weight_to_token_ratio=0.096241 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.109937674e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938033152 dispatch_meta_bytes=458752 combine_payload_bytes=938033152 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938491904 ingress_bytes=1114193920 link_bytes=1114193920 token_bytes=1876525056 weight_movement_bytes=176160768 total_nvlink_bytes=2052685824 weight_to_token_ratio=0.093876 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239020865e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131036 payload_rows_remote=114652 dup_rows=36 dup_rows_remote=0 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 18, 0, 0, 0, 0, 8174, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16348, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 36, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16348, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126498 payload_rows_remote=110142 dup_rows=4574 dup_rows_remote=160 dedup_ratio=0.034897 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 8189, 0, 0, 0, 8192, 17, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8170, 8, 8192] payload_rows_per_dst=[12464, 16252, 15962, 16384, 16384, 16312, 16384, 16356] dup_rows_per_dst=[0, 0, 134, 24, 0, 0, 2, 4414] combine_rows_per_src=[12464, 16252, 15962, 16384, 16384, 16312, 16384, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938655744 dispatch_meta_bytes=458800 combine_payload_bytes=938655744 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939114544 ingress_bytes=1643298816 link_bytes=1643298816 token_bytes=1877770288 weight_movement_bytes=704643072 total_nvlink_bytes=2582413360 weight_to_token_ratio=0.375255 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201787423e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28910192472 prefetch_bytes=4227858432 reduce_bytes=7222591488 weight_movement_bytes=11450449920 egress_bytes=14458718040 ingress_bytes=25901924352 link_bytes=25901924352 total_nvlink_bytes=40360642392 weight_to_token_ratio=0.396070 prefetch_share_of_ingress=0.163226 dedup_ratio=0.036287 prefetch_slots=48 unused_prefetch_slots=208 slot_utilization=0.187500 reduce_slots=41 remote_demand=24 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.022017 gemm_inner_pad_upper_bound=17653 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.064997773e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29145572688 prefetch_bytes=3523215360 reduce_bytes=4932501504 weight_movement_bytes=8455716864 egress_bytes=14576444752 ingress_bytes=23024844800 link_bytes=23024844800 total_nvlink_bytes=37601289552 weight_to_token_ratio=0.290120 prefetch_share_of_ingress=0.153018 dedup_ratio=0.029671 prefetch_slots=40 unused_prefetch_slots=216 slot_utilization=0.156250 reduce_slots=28 remote_demand=20 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.023201 gemm_inner_pad_upper_bound=16383 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.058094990e-05 prefetch_bytes_per_gemm_flop=1.233033460e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28686396312 prefetch_bytes=3523215360 reduce_bytes=6341787648 weight_movement_bytes=9865003008 egress_bytes=14346824600 ingress_bytes=24204574720 link_bytes=24204574720 total_nvlink_bytes=38551399320 weight_to_token_ratio=0.343891 prefetch_share_of_ingress=0.145560 dedup_ratio=0.040353 prefetch_slots=40 unused_prefetch_slots=216 slot_utilization=0.156250 reduce_slots=36 remote_demand=20 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.023082 gemm_inner_pad_upper_bound=16383 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.470969684e-05 prefetch_bytes_per_gemm_flop=1.233033460e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=458608 combine_payload_bytes=939229184 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939687792 ingress_bytes=1291550720 link_bytes=1291550720 token_bytes=1878916976 weight_movement_bytes=352321536 total_nvlink_bytes=2231238512 weight_to_token_ratio=0.187513 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232144912e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28611814448 prefetch_bytes=4580179968 reduce_bytes=2994733056 weight_movement_bytes=7574913024 egress_bytes=14309565488 ingress_bytes=21877161984 link_bytes=21877161984 total_nvlink_bytes=36186727472 weight_to_token_ratio=0.264748 prefetch_share_of_ingress=0.209359 dedup_ratio=0.044159 prefetch_slots=52 unused_prefetch_slots=204 slot_utilization=0.203125 reduce_slots=17 remote_demand=26 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.022017 gemm_inner_pad_upper_bound=17399 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.656435946e-05 prefetch_bytes_per_gemm_flop=1.602943497e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=902283264 dispatch_meta_bytes=441208 combine_payload_bytes=902283264 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=902724472 ingress_bytes=1606926336 link_bytes=1606926336 token_bytes=1805007736 weight_movement_bytes=704643072 total_nvlink_bytes=2509650808 weight_to_token_ratio=0.390382 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.998116718e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130162 payload_rows_remote=113780 dup_rows=910 dup_rows_remote=904 dedup_ratio=0.006943 router_tokens_per_expert=[5, 411, 8190, 8181, 2, 2, 8188, 11, 1, 0, 8184, 32, 1, 8187, 0, 0, 0, 1, 1, 0, 1, 2, 2, 0, 0, 0, 8191, 1, 0, 7747, 6, 8189] payload_rows_per_dst=[16380, 16380, 16372, 16376, 16384, 16382, 16384, 15504] dup_rows_per_dst=[812, 26, 62, 0, 2, 6, 0, 2] combine_rows_per_src=[16380, 16380, 16372, 16376, 16384, 16382, 16384, 15504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28090328024 prefetch_bytes=5108662272 reduce_bytes=1233125376 weight_movement_bytes=6341787648 egress_bytes=14048748504 ingress_bytes=20383367168 link_bytes=20383367168 total_nvlink_bytes=34432115672 weight_to_token_ratio=0.225764 prefetch_share_of_ingress=0.250629 dedup_ratio=0.058173 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=7 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.022017 gemm_inner_pad_upper_bound=17399 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.133646732e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=932085760 dispatch_meta_bytes=458736 combine_payload_bytes=932085760 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=932544496 ingress_bytes=1284407296 link_bytes=1284407296 token_bytes=1864630256 weight_movement_bytes=352321536 total_nvlink_bytes=2216951792 weight_to_token_ratio=0.188950 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.192144720e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130938 payload_rows_remote=114554 dup_rows=134 dup_rows_remote=134 dedup_ratio=0.001022 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 8183, 0, 0, 0, 8192, 67, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8124, 10, 8192] payload_rows_per_dst=[16384, 16366, 16384, 16384, 16384, 16384, 16384, 16268] dup_rows_per_dst=[0, 0, 134, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16366, 16384, 16384, 16384, 16384, 16384, 16268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29248950448 prefetch_bytes=4404019200 reduce_bytes=4932501504 weight_movement_bytes=9336520704 egress_bytes=14628147376 ingress_bytes=23957323776 link_bytes=23957782528 total_nvlink_bytes=38585471152 weight_to_token_ratio=0.319209 prefetch_share_of_ingress=0.183828 dedup_ratio=0.026320 prefetch_slots=50 unused_prefetch_slots=206 slot_utilization=0.195312 reduce_slots=28 remote_demand=25 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.021544 gemm_inner_pad_upper_bound=17907 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.384598855e-05 prefetch_bytes_per_gemm_flop=1.541291824e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938426368 dispatch_meta_bytes=458752 combine_payload_bytes=938426368 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938885120 ingress_bytes=1290747904 link_bytes=1290747904 token_bytes=1877311488 weight_movement_bytes=352321536 total_nvlink_bytes=2229633024 weight_to_token_ratio=0.187673 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227649478e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29275463696 prefetch_bytes=4932501504 reduce_bytes=3170893824 weight_movement_bytes=8103395328 egress_bytes=14641405968 ingress_bytes=22737453056 link_bytes=22737453056 total_nvlink_bytes=37378859024 weight_to_token_ratio=0.276798 prefetch_share_of_ingress=0.216933 dedup_ratio=0.024264 prefetch_slots=56 unused_prefetch_slots=200 slot_utilization=0.218750 reduce_slots=18 remote_demand=28 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.022372 gemm_inner_pad_upper_bound=17272 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.957515377e-05 prefetch_bytes_per_gemm_flop=1.726246843e-05 anomalies=none - [2026-09-16 21:37:07] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 566.5 | throughput per GPU (TFLOP/s/GPU): 588.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.558990E+00 | loss scale: 1.0 | grad norm: 5.842 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95052 payload_rows_remote=78962 dup_rows=36020 dup_rows_remote=24428 dedup_ratio=0.274811 router_tokens_per_expert=[3143, 17, 2500, 664, 2092, 25, 3200, 402, 7, 1319, 0, 1952, 2979, 2008, 320, 220, 2, 64, 2347, 3, 4, 1273, 5093, 229, 3, 3092, 8056, 2894, 5497, 5423, 3352, 7356] payload_rows_per_dst=[10330, 9312, 14026, 9304, 9364, 11972, 16090, 14654] dup_rows_per_dst=[2318, 2126, 4620, 1750, 2266, 1634, 11592, 9714] combine_rows_per_src=[10330, 9312, 14026, 9304, 9364, 11972, 16090, 14654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95998 payload_rows_remote=81560 dup_rows=35074 dup_rows_remote=26444 dedup_ratio=0.267593 router_tokens_per_expert=[3614, 0, 2271, 912, 2304, 1, 3566, 194, 22, 1678, 0, 2267, 2370, 2917, 112, 210, 0, 30, 3069, 0, 0, 1510, 5097, 164, 0, 3065, 8071, 3083, 4685, 5293, 2946, 6085] payload_rows_per_dst=[12446, 14438, 13838, 13246, 14136, 12080, 9922, 5892] dup_rows_per_dst=[6352, 8630, 5258, 6766, 4232, 1462, 2374, 0] combine_rows_per_src=[12446, 14438, 13838, 13246, 14136, 12080, 9922, 5892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97704 payload_rows_remote=85280 dup_rows=33368 dup_rows_remote=29954 dedup_ratio=0.254578 router_tokens_per_expert=[3993, 4, 2261, 626, 1880, 1, 3221, 522, 35, 1186, 0, 1584, 2769, 3474, 270, 214, 1, 3, 3123, 3, 4, 1198, 4886, 210, 1, 3109, 8131, 2585, 4957, 4827, 3642, 6816] payload_rows_per_dst=[10972, 16352, 12424, 13486, 14806, 11454, 9574, 8636] dup_rows_per_dst=[2796, 11158, 3414, 7296, 5086, 1142, 1816, 660] combine_rows_per_src=[10972, 16352, 12424, 13486, 14806, 11454, 9574, 8636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94726 payload_rows_remote=78388 dup_rows=36346 dup_rows_remote=24282 dedup_ratio=0.277298 router_tokens_per_expert=[3138, 1, 2197, 779, 1655, 1, 3241, 398, 5, 1247, 1, 2493, 3361, 2486, 159, 234, 5, 0, 2449, 1, 2, 903, 5461, 185, 10, 3384, 8086, 2752, 4706, 5451, 3188, 7557] payload_rows_per_dst=[16338, 8872, 13218, 14080, 15654, 12086, 11738, 2740] dup_rows_per_dst=[12064, 1718, 4310, 8678, 4370, 1016, 4190, 0] combine_rows_per_src=[16338, 8872, 13218, 14080, 15654, 12086, 11738, 2740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97296 payload_rows_remote=81496 dup_rows=33776 dup_rows_remote=21654 dedup_ratio=0.257690 router_tokens_per_expert=[3101, 4, 2837, 729, 1787, 5, 2980, 183, 9, 1664, 0, 1611, 3470, 3648, 116, 178, 0, 0, 2469, 0, 0, 1102, 5365, 293, 2, 2149, 8081, 3065, 5455, 5172, 3770, 6291] payload_rows_per_dst=[10856, 8554, 13448, 11440, 13992, 15800, 9732, 13474] dup_rows_per_dst=[2486, 1356, 4220, 3384, 3528, 12122, 2460, 4220] combine_rows_per_src=[10856, 8554, 13448, 11440, 13992, 15800, 9732, 13474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 2, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98136 payload_rows_remote=86868 dup_rows=32936 dup_rows_remote=29164 dedup_ratio=0.251282 router_tokens_per_expert=[3781, 0, 2054, 813, 2134, 1, 3893, 371, 11, 1680, 8, 2243, 3219, 2987, 199, 157, 6, 4, 2376, 0, 5, 873, 4472, 110, 2, 3060, 7936, 2809, 5234, 5137, 3470, 6491] payload_rows_per_dst=[10654, 16034, 13314, 11268, 14358, 10266, 9898, 12344] dup_rows_per_dst=[2642, 12104, 4714, 3772, 3396, 1186, 1844, 3278] combine_rows_per_src=[10654, 16034, 13314, 11268, 14358, 10266, 9898, 12344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97760 payload_rows_remote=82734 dup_rows=33312 dup_rows_remote=27586 dedup_ratio=0.254150 router_tokens_per_expert=[3467, 1, 3143, 719, 1684, 3, 2833, 303, 4, 1531, 0, 1727, 2949, 3324, 168, 245, 8, 7, 3648, 0, 1, 849, 4531, 98, 0, 3198, 8087, 2980, 5670, 4654, 2991, 6713] payload_rows_per_dst=[11474, 8208, 13356, 10844, 15026, 16326, 10238, 12288] dup_rows_per_dst=[3186, 1472, 4200, 2528, 5726, 10772, 2118, 3310] combine_rows_per_src=[11474, 8208, 13356, 10844, 15026, 16326, 10238, 12288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=646856704 dispatch_meta_bytes=413560 combine_payload_bytes=646856704 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=647270264 ingress_bytes=2056142848 link_bytes=2056142848 token_bytes=1294126968 weight_movement_bytes=1409286144 total_nvlink_bytes=2703413112 weight_to_token_ratio=1.088986 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=12160 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.151354167e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=85862 payload_rows_remote=69772 dup_rows=45210 dup_rows_remote=21438 dedup_ratio=0.344925 router_tokens_per_expert=[3989, 16, 3332, 742, 1928, 0, 3005, 436, 19, 1631, 0, 1585, 2767, 3310, 317, 221, 1, 8, 2557, 0, 3, 1228, 4374, 272, 2, 2775, 8055, 2911, 4421, 5156, 3519, 6956] payload_rows_per_dst=[12026, 8798, 6168, 10718, 5130, 10660, 16272, 16090] dup_rows_per_dst=[4132, 1940, 544, 2512, 2, 1094, 11214, 23772] combine_rows_per_src=[12026, 8798, 6168, 10718, 5130, 10660, 16272, 16090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=668139520 dispatch_meta_bytes=432016 combine_payload_bytes=668139520 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=668571536 ingress_bytes=1020461056 link_bytes=1020461056 token_bytes=1336711056 weight_movement_bytes=352321536 total_nvlink_bytes=1689032592 weight_to_token_ratio=0.263573 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=13184 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.714155952e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=642154496 dispatch_meta_bytes=410680 combine_payload_bytes=642154496 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=642565176 ingress_bytes=994476032 link_bytes=994476032 token_bytes=1284719672 weight_movement_bytes=352321536 total_nvlink_bytes=1637041208 weight_to_token_ratio=0.274240 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=14592 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.568650664e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=698613760 dispatch_meta_bytes=460936 combine_payload_bytes=698613760 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=699074696 ingress_bytes=1050935296 link_bytes=1050935296 token_bytes=1397688456 weight_movement_bytes=352321536 total_nvlink_bytes=1750009992 weight_to_token_ratio=0.252074 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=19200 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.884798976e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=667615232 dispatch_meta_bytes=412600 combine_payload_bytes=667615232 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=668027832 ingress_bytes=1019936768 link_bytes=1019936768 token_bytes=1335643064 weight_movement_bytes=352321536 total_nvlink_bytes=1687964600 weight_to_token_ratio=0.263784 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=20224 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.711220158e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=711622656 dispatch_meta_bytes=464128 combine_payload_bytes=711622656 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=712086784 ingress_bytes=1063944192 link_bytes=1063944192 token_bytes=1423709440 weight_movement_bytes=352321536 total_nvlink_bytes=1776030976 weight_to_token_ratio=0.247467 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=12416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.957643363e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=677756928 dispatch_meta_bytes=441280 combine_payload_bytes=677756928 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=678198208 ingress_bytes=1030078464 link_bytes=1030078464 token_bytes=1355955136 weight_movement_bytes=352321536 total_nvlink_bytes=1708276672 weight_to_token_ratio=0.259833 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=21888 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.768009422e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=571572224 dispatch_meta_bytes=364840 combine_payload_bytes=571572224 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=571937064 ingress_bytes=1980858368 link_bytes=1980858368 token_bytes=1143509288 weight_movement_bytes=1409286144 total_nvlink_bytes=2552795432 weight_to_token_ratio=1.232422 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.109198001e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116344 payload_rows_remote=99960 dup_rows=14728 dup_rows_remote=14530 dedup_ratio=0.112366 router_tokens_per_expert=[7993, 3, 2040, 0, 8147, 0, 7, 0, 8192, 5954, 99, 0, 5755, 0, 2675, 8159, 8176, 2, 1, 0, 2, 0, 0, 0, 0, 21, 62, 16, 4, 8171, 13, 44] payload_rows_per_dst=[4084, 16308, 16344, 14160, 16358, 16322, 16384, 16384] dup_rows_per_dst=[2, 0, 11748, 2700, 0, 0, 198, 80] combine_rows_per_src=[4084, 16308, 16344, 14160, 16358, 16322, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123264 payload_rows_remote=106880 dup_rows=7808 dup_rows_remote=7808 dedup_ratio=0.059570 router_tokens_per_expert=[8106, 0, 2610, 0, 8192, 0, 0, 0, 8192, 5816, 42, 2, 5893, 0, 2106, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16312, 16384, 11698, 13334, 16384, 16384, 16384, 16384] dup_rows_per_dst=[5994, 0, 22, 1790, 0, 0, 2, 0] combine_rows_per_src=[16312, 16384, 11698, 13334, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126606 payload_rows_remote=113324 dup_rows=4466 dup_rows_remote=4444 dedup_ratio=0.034073 router_tokens_per_expert=[8051, 1, 1162, 3, 8191, 0, 0, 1, 8186, 6636, 16, 0, 6612, 1, 2097, 8185, 8191, 2, 1, 0, 0, 0, 1, 0, 0, 5, 2, 0, 0, 8188, 1, 3] payload_rows_per_dst=[16244, 16384, 13282, 15184, 16384, 16370, 16376, 16382] dup_rows_per_dst=[2190, 0, 22, 2236, 4, 2, 10, 2] combine_rows_per_src=[16244, 16384, 13282, 15184, 16384, 16370, 16376, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117722 payload_rows_remote=101338 dup_rows=13350 dup_rows_remote=520 dedup_ratio=0.101852 router_tokens_per_expert=[8113, 0, 711, 2, 8187, 0, 0, 0, 8191, 5652, 158, 0, 5781, 4, 4158, 8181, 8192, 0, 0, 0, 0, 0, 1, 1, 1, 0, 2, 0, 0, 8191, 2, 8] payload_rows_per_dst=[16384, 16382, 11422, 8316, 16384, 16364, 16186, 16284] dup_rows_per_dst=[12830, 98, 398, 12, 0, 2, 2, 8] combine_rows_per_src=[16384, 16382, 11422, 8316, 16384, 16364, 16186, 16284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 3, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=818872320 dispatch_meta_bytes=457960 combine_payload_bytes=818872320 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=819330280 ingress_bytes=1347354624 link_bytes=1347354624 token_bytes=1638202600 weight_movement_bytes=528482304 total_nvlink_bytes=2166684904 weight_to_token_ratio=0.322599 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.544623481e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120998 payload_rows_remote=104614 dup_rows=10074 dup_rows_remote=10072 dedup_ratio=0.076859 router_tokens_per_expert=[8071, 0, 2347, 0, 8180, 0, 0, 0, 8191, 5129, 102, 0, 6345, 3, 2580, 8192, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 8188, 0, 13] payload_rows_per_dst=[10064, 16360, 14108, 14932, 16384, 16384, 16382, 16384] dup_rows_per_dst=[1406, 0, 5720, 2924, 0, 2, 4, 18] combine_rows_per_src=[10064, 16360, 14108, 14932, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=875560960 dispatch_meta_bytes=458752 combine_payload_bytes=875560960 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=876019712 ingress_bytes=1227882496 link_bytes=1227882496 token_bytes=1751580672 weight_movement_bytes=352321536 total_nvlink_bytes=2103902208 weight_to_token_ratio=0.201145 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.875629434e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125306 payload_rows_remote=108928 dup_rows=5766 dup_rows_remote=5766 dedup_ratio=0.043991 router_tokens_per_expert=[8083, 0, 1292, 0, 8192, 0, 0, 0, 8189, 5917, 49, 7, 6633, 3, 2599, 8191, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 8185, 0, 1] payload_rows_per_dst=[16244, 16384, 11852, 15310, 16378, 16382, 16384, 16372] dup_rows_per_dst=[2506, 0, 94, 3160, 0, 0, 6, 0] combine_rows_per_src=[16244, 16384, 11852, 15310, 16378, 16382, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=928350208 dispatch_meta_bytes=471072 combine_payload_bytes=928350208 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=928821280 ingress_bytes=1809154048 link_bytes=1809154048 token_bytes=1857171488 weight_movement_bytes=880803840 total_nvlink_bytes=2737975328 weight_to_token_ratio=0.474272 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.013050749e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=830160896 dispatch_meta_bytes=407432 combine_payload_bytes=830160896 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=830568328 ingress_bytes=1534803968 link_bytes=1534803968 token_bytes=1660729224 weight_movement_bytes=704643072 total_nvlink_bytes=2365372296 weight_to_token_ratio=0.424297 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=22528 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.594261562e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121918 payload_rows_remote=106994 dup_rows=9154 dup_rows_remote=4894 dedup_ratio=0.069839 router_tokens_per_expert=[7959, 0, 2574, 4, 8191, 0, 0, 3, 8189, 4427, 29, 6, 6812, 0, 2780, 8192, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8181, 0, 0] payload_rows_per_dst=[16198, 16384, 8912, 14924, 16376, 16384, 16378, 16362] dup_rows_per_dst=[4876, 4, 12, 4260, 0, 0, 2, 0] combine_rows_per_src=[16198, 16384, 8912, 14924, 16376, 16384, 16378, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118564 payload_rows_remote=102180 dup_rows=12508 dup_rows_remote=12480 dedup_ratio=0.095428 router_tokens_per_expert=[7916, 0, 2861, 8, 8192, 0, 0, 0, 8174, 4796, 19, 0, 6988, 5, 2001, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[5738, 16384, 16276, 14668, 16384, 16382, 16348, 16384] dup_rows_per_dst=[0, 0, 9162, 3162, 36, 94, 26, 28] combine_rows_per_src=[5738, 16384, 16276, 14668, 16384, 16382, 16348, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=856997888 dispatch_meta_bytes=458744 combine_payload_bytes=856997888 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=857456632 ingress_bytes=1561640960 link_bytes=1561640960 token_bytes=1714454520 weight_movement_bytes=704643072 total_nvlink_bytes=2419097592 weight_to_token_ratio=0.411001 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.744537515e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=892338176 dispatch_meta_bytes=458776 combine_payload_bytes=892338176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=892796952 ingress_bytes=1244659712 link_bytes=1244659712 token_bytes=1785135128 weight_movement_bytes=352321536 total_nvlink_bytes=2137456664 weight_to_token_ratio=0.197364 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.969574841e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 2, 4, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120338 payload_rows_remote=103958 dup_rows=10734 dup_rows_remote=10732 dedup_ratio=0.081894 router_tokens_per_expert=[40, 32, 8185, 3, 0, 0, 2792, 0, 8131, 8188, 5140, 6, 8189, 1, 3, 0, 10, 4, 0, 3, 0, 0, 8182, 8162, 5, 0, 0, 8186, 7, 226, 0, 41] payload_rows_per_dst=[16380, 6474, 15632, 16382, 16384, 16324, 16380, 16382] dup_rows_per_dst=[140, 590, 9442, 4, 26, 0, 2, 530] combine_rows_per_src=[16380, 6474, 15632, 16382, 16384, 16324, 16380, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=837058560 dispatch_meta_bytes=458640 combine_payload_bytes=837058560 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=837517200 ingress_bytes=1541701632 link_bytes=1541701632 token_bytes=1674575760 weight_movement_bytes=704643072 total_nvlink_bytes=2379218832 weight_to_token_ratio=0.420789 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.632885601e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=876494848 dispatch_meta_bytes=447552 combine_payload_bytes=876494848 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=876942400 ingress_bytes=2638102528 link_bytes=2638102528 token_bytes=1753437248 weight_movement_bytes=1761607680 total_nvlink_bytes=3515044928 weight_to_token_ratio=1.004660 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=23296 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.477227296e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127644 payload_rows_remote=111316 dup_rows=3428 dup_rows_remote=646 dedup_ratio=0.026154 router_tokens_per_expert=[2, 17, 8191, 0, 0, 0, 1465, 0, 8090, 8192, 6526, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8172, 0, 305, 0, 0] payload_rows_per_dst=[16384, 16328, 13052, 16384, 16384, 16384, 16344, 16384] dup_rows_per_dst=[36, 2782, 0, 0, 0, 0, 0, 610] combine_rows_per_src=[16384, 16328, 13052, 16384, 16384, 16384, 16344, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127760 payload_rows_remote=114644 dup_rows=3312 dup_rows_remote=3312 dedup_ratio=0.025269 router_tokens_per_expert=[4, 8, 8191, 1, 2, 0, 1236, 0, 8164, 8191, 6556, 2, 8188, 0, 1, 0, 2, 1, 0, 1, 0, 1, 8191, 8185, 0, 0, 0, 8188, 0, 421, 0, 2] payload_rows_per_dst=[16384, 16366, 13116, 16378, 16384, 16372, 16376, 16384] dup_rows_per_dst=[24, 2438, 0, 0, 6, 0, 0, 844] combine_rows_per_src=[16384, 16366, 13116, 16378, 16384, 16372, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123674 payload_rows_remote=107290 dup_rows=7398 dup_rows_remote=7378 dedup_ratio=0.056442 router_tokens_per_expert=[0, 10, 8192, 0, 1, 0, 2141, 0, 8115, 8192, 5939, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 2, 8177, 0, 192, 0, 2] payload_rows_per_dst=[16384, 16376, 13908, 16382, 16352, 16088, 16358, 11826] dup_rows_per_dst=[20, 4138, 3054, 0, 0, 0, 0, 186] combine_rows_per_src=[16384, 16376, 13908, 16382, 16352, 16088, 16358, 11826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=851623936 dispatch_meta_bytes=458760 combine_payload_bytes=851623936 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=852082696 ingress_bytes=1203945472 link_bytes=1203945472 token_bytes=1703706632 weight_movement_bytes=352321536 total_nvlink_bytes=2056028168 weight_to_token_ratio=0.206797 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.741592092e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=911900672 dispatch_meta_bytes=447848 combine_payload_bytes=911900672 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=912348520 ingress_bytes=1264222208 link_bytes=1264222208 token_bytes=1824249192 weight_movement_bytes=352321536 total_nvlink_bytes=2176570728 weight_to_token_ratio=0.193132 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=24960 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.079116652e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124780 payload_rows_remote=108400 dup_rows=6292 dup_rows_remote=6292 dedup_ratio=0.048004 router_tokens_per_expert=[6, 38, 8192, 0, 0, 0, 2460, 0, 8060, 8192, 5580, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 8183, 0, 250, 0, 1] payload_rows_per_dst=[16384, 15124, 11374, 16384, 16384, 16380, 16366, 16384] dup_rows_per_dst=[88, 4510, 1192, 0, 0, 0, 0, 502] combine_rows_per_src=[16384, 15124, 11374, 16384, 16384, 16380, 16366, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=471824 combine_payload_bytes=939163648 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939635472 ingress_bytes=1996128256 link_bytes=1996128256 token_bytes=1878799120 weight_movement_bytes=1056964608 total_nvlink_bytes=2935763728 weight_to_token_ratio=0.562575 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=24192 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117748501e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125956 payload_rows_remote=109572 dup_rows=5116 dup_rows_remote=5116 dedup_ratio=0.039032 router_tokens_per_expert=[6, 29, 8192, 0, 0, 0, 2183, 0, 8127, 8192, 5672, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 8188, 0, 372, 0, 3] payload_rows_per_dst=[16384, 16322, 11344, 16380, 16384, 16382, 16376, 16384] dup_rows_per_dst=[70, 4298, 0, 0, 0, 0, 0, 748] combine_rows_per_src=[16384, 16322, 11344, 16380, 16384, 16382, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=878919680 dispatch_meta_bytes=458672 combine_payload_bytes=878919680 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=879378352 ingress_bytes=1935884288 link_bytes=1935884288 token_bytes=1758298032 weight_movement_bytes=1056964608 total_nvlink_bytes=2815262640 weight_to_token_ratio=0.601129 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.084014393e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119870 payload_rows_remote=103486 dup_rows=11202 dup_rows_remote=10448 dedup_ratio=0.085464 router_tokens_per_expert=[2, 41, 8190, 0, 0, 0, 2932, 0, 7944, 8192, 5100, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8182, 0, 377, 0, 0] payload_rows_per_dst=[15858, 5864, 16248, 16384, 16384, 16384, 16364, 16384] dup_rows_per_dst=[82, 0, 9840, 8, 0, 374, 144, 754] combine_rows_per_src=[15858, 5864, 16248, 16384, 16384, 16384, 16364, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127474 payload_rows_remote=111090 dup_rows=3598 dup_rows_remote=3596 dedup_ratio=0.027451 router_tokens_per_expert=[26, 9, 8192, 0, 0, 1, 1687, 0, 8043, 8192, 6435, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 8181, 0, 195, 0, 0] payload_rows_per_dst=[16384, 16326, 12870, 16384, 16384, 16384, 16362, 16380] dup_rows_per_dst=[70, 3136, 0, 2, 0, 0, 0, 390] combine_rows_per_src=[16384, 16326, 12870, 16384, 16384, 16384, 16362, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=888012800 dispatch_meta_bytes=458768 combine_payload_bytes=888012800 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=888471568 ingress_bytes=1944977408 link_bytes=1944977408 token_bytes=1776484368 weight_movement_bytes=1056964608 total_nvlink_bytes=2833448976 weight_to_token_ratio=0.594975 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.024621 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.089106161e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=897613824 dispatch_meta_bytes=458752 combine_payload_bytes=897613824 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=898072576 ingress_bytes=1249935360 link_bytes=1249935360 token_bytes=1795686400 weight_movement_bytes=352321536 total_nvlink_bytes=2148007936 weight_to_token_ratio=0.196204 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.999116267e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116080 payload_rows_remote=99788 dup_rows=14992 dup_rows_remote=1868 dedup_ratio=0.114380 router_tokens_per_expert=[4, 1, 0, 4, 4, 1, 2, 1046, 8166, 8186, 2, 0, 3, 0, 1, 3, 8188, 4, 4, 8189, 14, 2, 0, 2511, 4675, 8187, 0, 8104, 0, 4, 8186, 45] payload_rows_per_dst=[16384, 12886, 16340, 16384, 16384, 5026, 16292, 16384] dup_rows_per_dst=[10, 1408, 322, 4, 10, 28, 13124, 86] combine_rows_per_src=[16384, 12886, 16340, 16384, 16384, 5026, 16292, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122430 payload_rows_remote=106046 dup_rows=8642 dup_rows_remote=4932 dedup_ratio=0.065933 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1855, 8190, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8184, 9, 0, 0, 2482, 3881, 8192, 0, 8167, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16368, 16384, 7762, 16384] dup_rows_per_dst=[0, 3710, 0, 0, 0, 4932, 0, 0] combine_rows_per_src=[16384, 16384, 16380, 16384, 16368, 16384, 7762, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=847757312 dispatch_meta_bytes=455736 combine_payload_bytes=847757312 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=848213048 ingress_bytes=1200078848 link_bytes=1200078848 token_bytes=1695970360 weight_movement_bytes=352321536 total_nvlink_bytes=2048291896 weight_to_token_ratio=0.207740 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.719940611e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122354 payload_rows_remote=106002 dup_rows=8718 dup_rows_remote=8718 dedup_ratio=0.066513 router_tokens_per_expert=[1, 0, 0, 1, 1, 0, 0, 1312, 8175, 8189, 1, 0, 1, 0, 1, 0, 8190, 3, 1, 8184, 7, 0, 0, 3057, 3860, 8191, 10, 8160, 0, 1, 8188, 2] payload_rows_per_dst=[16382, 16380, 16352, 16384, 16376, 16382, 7720, 16378] dup_rows_per_dst=[2, 2624, 0, 2, 0, 6066, 20, 4] combine_rows_per_src=[16382, 16380, 16352, 16384, 16376, 16382, 7720, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=910049280 dispatch_meta_bytes=458744 combine_payload_bytes=910049280 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=910508024 ingress_bytes=1262370816 link_bytes=1262370816 token_bytes=1820557304 weight_movement_bytes=352321536 total_nvlink_bytes=2172878840 weight_to_token_ratio=0.193524 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.068749630e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=817463296 dispatch_meta_bytes=406624 combine_payload_bytes=817463296 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=817869920 ingress_bytes=1874427904 link_bytes=1874427904 token_bytes=1635333216 weight_movement_bytes=1056964608 total_nvlink_bytes=2692297824 weight_to_token_ratio=0.646330 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=15104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.049601384e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=868728832 dispatch_meta_bytes=443912 combine_payload_bytes=868728832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=869172744 ingress_bytes=1221050368 link_bytes=1221050368 token_bytes=1737901576 weight_movement_bytes=352321536 total_nvlink_bytes=2090223112 weight_to_token_ratio=0.202728 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.837372369e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123266 payload_rows_remote=106882 dup_rows=7806 dup_rows_remote=7806 dedup_ratio=0.059555 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 923, 8184, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 28, 0, 0, 2912, 4379, 8192, 0, 8153, 0, 0, 8192, 3] payload_rows_per_dst=[16384, 16384, 16368, 16384, 16380, 16384, 8758, 16224] dup_rows_per_dst=[0, 1846, 66, 0, 92, 5802, 0, 0] combine_rows_per_src=[16384, 16384, 16368, 16384, 16380, 16384, 8758, 16224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=868368384 dispatch_meta_bytes=458880 combine_payload_bytes=868368384 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=868827264 ingress_bytes=1925332992 link_bytes=1925332992 token_bytes=1737195648 weight_movement_bytes=1056964608 total_nvlink_bytes=2794160256 weight_to_token_ratio=0.608432 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.078106108e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124116 payload_rows_remote=114560 dup_rows=6956 dup_rows_remote=6876 dedup_ratio=0.053070 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 804, 8154, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8176, 40, 0, 0, 4778, 2709, 8192, 0, 8099, 0, 0, 8192, 8] payload_rows_per_dst=[16384, 16384, 16308, 16384, 16352, 9556, 16364, 16384] dup_rows_per_dst=[0, 1608, 0, 0, 0, 80, 5252, 16] combine_rows_per_src=[16384, 16384, 16308, 16384, 16352, 9556, 16364, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121698 payload_rows_remote=105320 dup_rows=9374 dup_rows_remote=9374 dedup_ratio=0.071518 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1841, 8188, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 15, 0, 0, 3114, 3261, 8192, 0, 8160, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16378, 14344, 9064, 16384] dup_rows_per_dst=[0, 3682, 0, 0, 0, 4806, 886, 0] combine_rows_per_src=[16384, 16384, 16376, 16384, 16378, 14344, 9064, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=875577344 dispatch_meta_bytes=458752 combine_payload_bytes=875577344 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=876036096 ingress_bytes=1227898880 link_bytes=1227898880 token_bytes=1751613440 weight_movement_bytes=352321536 total_nvlink_bytes=2103934976 weight_to_token_ratio=0.201141 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.875721178e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=109044 payload_rows_remote=92660 dup_rows=22028 dup_rows_remote=21978 dedup_ratio=0.168060 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1735, 8178, 8175, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 30, 0, 0, 3666, 2839, 8192, 1, 8138, 0, 0, 8192, 12] payload_rows_per_dst=[16374, 3470, 16350, 16378, 16372, 7332, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 10, 60, 21908, 50] combine_rows_per_src=[16374, 3470, 16350, 16378, 16372, 7332, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938475520 dispatch_meta_bytes=485744 combine_payload_bytes=938475520 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938961264 ingress_bytes=1290797056 link_bytes=1290797056 token_bytes=1877436784 weight_movement_bytes=352321536 total_nvlink_bytes=2229758320 weight_to_token_ratio=0.187661 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227924708e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122018 payload_rows_remote=105634 dup_rows=9054 dup_rows_remote=9054 dedup_ratio=0.069077 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 769, 8105, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8175, 10, 0, 0, 3814, 3768, 8192, 1, 8121, 0, 0, 8192, 5] payload_rows_per_dst=[16384, 16384, 16210, 16384, 16350, 16384, 7538, 16384] dup_rows_per_dst=[0, 1538, 0, 0, 0, 7506, 0, 10] combine_rows_per_src=[16384, 16384, 16210, 16384, 16350, 16384, 7538, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120386 payload_rows_remote=104012 dup_rows=10686 dup_rows_remote=10680 dedup_ratio=0.081528 router_tokens_per_expert=[4, 2, 4, 0, 2880, 8183, 8188, 4, 0, 0, 7, 11, 4, 4, 2, 8187, 22, 0, 1, 0, 4, 5105, 198, 8170, 0, 0, 8187, 3, 8177, 1, 8188, 0] payload_rows_per_dst=[16384, 5768, 16360, 16384, 16376, 16384, 16374, 16356] dup_rows_per_dst=[12, 0, 16, 10, 46, 10596, 6, 0] combine_rows_per_src=[16384, 5768, 16360, 16384, 16376, 16384, 16374, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123992 payload_rows_remote=107608 dup_rows=7080 dup_rows_remote=20 dedup_ratio=0.054016 router_tokens_per_expert=[0, 0, 0, 0, 3531, 8191, 8192, 0, 1, 7, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4594, 104, 8192, 0, 0, 8192, 0, 8154, 0, 8184, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16368, 9396, 16384, 16308] dup_rows_per_dst=[0, 7060, 20, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16368, 9396, 16384, 16308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=862781440 dispatch_meta_bytes=458776 combine_payload_bytes=862781440 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=863240216 ingress_bytes=1567424512 link_bytes=1567424512 token_bytes=1726021656 weight_movement_bytes=704643072 total_nvlink_bytes=2430664728 weight_to_token_ratio=0.408247 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.776922992e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122220 payload_rows_remote=105842 dup_rows=8852 dup_rows_remote=8830 dedup_ratio=0.067535 router_tokens_per_expert=[1, 0, 1, 0, 4405, 8191, 8191, 1, 2, 0, 0, 13, 1, 0, 1, 8191, 2, 2, 0, 0, 1, 3769, 42, 8185, 1, 4, 8191, 0, 8148, 1, 8185, 7] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16374, 7620, 16384, 16312] dup_rows_per_dst=[2, 8810, 22, 2, 4, 4, 8, 0] combine_rows_per_src=[16384, 16384, 16378, 16384, 16374, 7620, 16384, 16312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=759070720 dispatch_meta_bytes=458552 combine_payload_bytes=759070720 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=759529272 ingress_bytes=1463713792 link_bytes=1463713792 token_bytes=1518599992 weight_movement_bytes=704643072 total_nvlink_bytes=2223243064 weight_to_token_ratio=0.464008 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.196186251e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=852066304 dispatch_meta_bytes=458768 combine_payload_bytes=852066304 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=852525072 ingress_bytes=1556709376 link_bytes=1556709376 token_bytes=1704591376 weight_movement_bytes=704643072 total_nvlink_bytes=2409234448 weight_to_token_ratio=0.413379 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.716922703e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=865353728 dispatch_meta_bytes=458752 combine_payload_bytes=865353728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=865812480 ingress_bytes=1217675264 link_bytes=1217675264 token_bytes=1731166208 weight_movement_bytes=352321536 total_nvlink_bytes=2083487744 weight_to_token_ratio=0.203517 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.818473196e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120764 payload_rows_remote=104380 dup_rows=10308 dup_rows_remote=10308 dedup_ratio=0.078644 router_tokens_per_expert=[0, 0, 0, 0, 5101, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 3021, 97, 8192, 0, 8, 8192, 0, 8156, 0, 8189, 3] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 6236, 16330, 16280] dup_rows_per_dst=[0, 10270, 0, 0, 38, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 6236, 16330, 16280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=881524736 dispatch_meta_bytes=430512 combine_payload_bytes=881524736 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=881955248 ingress_bytes=2290810880 link_bytes=2290810880 token_bytes=1763479984 weight_movement_bytes=1409286144 total_nvlink_bytes=3172766128 weight_to_token_ratio=0.799151 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=18048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.282758470e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=867057664 dispatch_meta_bytes=458688 combine_payload_bytes=867057664 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=867516352 ingress_bytes=1219379200 link_bytes=1219379200 token_bytes=1734574016 weight_movement_bytes=352321536 total_nvlink_bytes=2086895552 weight_to_token_ratio=0.203117 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.828014526e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123950 payload_rows_remote=107566 dup_rows=7122 dup_rows_remote=76 dedup_ratio=0.054337 router_tokens_per_expert=[0, 0, 0, 0, 4665, 8192, 8192, 0, 23, 11, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3286, 237, 8184, 0, 11, 8192, 0, 8163, 0, 8179, 9] payload_rows_per_dst=[16384, 9330, 16384, 16384, 16358, 16384, 16384, 16342] dup_rows_per_dst=[0, 0, 52, 0, 0, 7046, 22, 2] combine_rows_per_src=[16384, 9330, 16384, 16384, 16358, 16384, 16384, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=855080960 dispatch_meta_bytes=458752 combine_payload_bytes=855080960 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=855539712 ingress_bytes=1207402496 link_bytes=1207402496 token_bytes=1710620672 weight_movement_bytes=352321536 total_nvlink_bytes=2062942208 weight_to_token_ratio=0.205961 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.760949983e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122176 payload_rows_remote=105800 dup_rows=8896 dup_rows_remote=8896 dedup_ratio=0.067871 router_tokens_per_expert=[0, 0, 0, 0, 4281, 8192, 8192, 0, 0, 2, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3638, 283, 8190, 0, 4, 8192, 0, 8176, 0, 8188, 5] payload_rows_per_dst=[16384, 10622, 16384, 16384, 16376, 13280, 16384, 16362] dup_rows_per_dst=[0, 3124, 2, 0, 0, 5762, 8, 0] combine_rows_per_src=[16384, 10622, 16384, 16384, 16376, 13280, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=881180672 dispatch_meta_bytes=430568 combine_payload_bytes=881180672 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=881611240 ingress_bytes=1585823744 link_bytes=1585823744 token_bytes=1762791912 weight_movement_bytes=704643072 total_nvlink_bytes=2467434984 weight_to_token_ratio=0.399731 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=17024 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.879951011e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118752 payload_rows_remote=102392 dup_rows=12320 dup_rows_remote=11908 dedup_ratio=0.093994 router_tokens_per_expert=[0, 0, 0, 0, 2155, 8192, 8192, 0, 2, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5472, 569, 8192, 0, 0, 8192, 0, 8180, 0, 8191, 0] payload_rows_per_dst=[16370, 4324, 16276, 16372, 16376, 16290, 16384, 16360] dup_rows_per_dst=[0, 0, 18, 0, 0, 11878, 12, 412] combine_rows_per_src=[16370, 4324, 16276, 16372, 16376, 16290, 16384, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122818 payload_rows_remote=106452 dup_rows=8254 dup_rows_remote=8252 dedup_ratio=0.062973 router_tokens_per_expert=[7741, 24, 8187, 0, 8166, 4, 2069, 4, 5, 14, 0, 5, 1, 0, 36, 0, 4, 8147, 6, 5, 8187, 0, 0, 6569, 0, 6, 8173, 8178, 0, 5, 0, 0] payload_rows_per_dst=[15530, 11818, 16384, 16374, 16306, 13684, 16366, 16356] dup_rows_per_dst=[0, 3500, 38, 74, 370, 4270, 2, 0] combine_rows_per_src=[15530, 11818, 16384, 16374, 16306, 13684, 16366, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124828 payload_rows_remote=108444 dup_rows=6244 dup_rows_remote=6244 dedup_ratio=0.047638 router_tokens_per_expert=[0, 0, 0, 0, 3114, 8192, 8192, 0, 0, 1, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4710, 388, 8192, 0, 4, 8192, 0, 8161, 0, 8187, 8] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16374, 10196, 16384, 16338] dup_rows_per_dst=[0, 6228, 8, 0, 0, 0, 8, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16374, 10196, 16384, 16338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128834 payload_rows_remote=112450 dup_rows=2238 dup_rows_remote=38 dedup_ratio=0.017075 router_tokens_per_expert=[8077, 3, 8183, 0, 8192, 2, 1098, 0, 0, 1, 0, 0, 0, 0, 11, 7, 0, 8178, 0, 0, 8192, 0, 0, 7221, 0, 0, 8192, 8179, 0, 0, 0, 0] payload_rows_per_dst=[16160, 16384, 16366, 16384, 16356, 14442, 16358, 16384] dup_rows_per_dst=[0, 2200, 2, 36, 0, 0, 0, 0] combine_rows_per_src=[16160, 16384, 16366, 16384, 16356, 14442, 16358, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=866713600 dispatch_meta_bytes=458784 combine_payload_bytes=866713600 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=867172384 ingress_bytes=1395195904 link_bytes=1395195904 token_bytes=1733885984 weight_movement_bytes=528482304 total_nvlink_bytes=2262368288 weight_to_token_ratio=0.304796 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.812514679e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128466 payload_rows_remote=112094 dup_rows=2606 dup_rows_remote=2588 dedup_ratio=0.019882 router_tokens_per_expert=[8132, 6, 8185, 0, 8191, 9, 1262, 1, 1, 8, 0, 1, 0, 4, 15, 2, 1, 8171, 1, 1, 8191, 0, 0, 6988, 0, 1, 8189, 8174, 0, 1, 1, 0] payload_rows_per_dst=[16276, 16384, 16372, 16382, 16344, 13976, 16350, 16382] dup_rows_per_dst=[0, 2542, 18, 42, 4, 0, 0, 0] combine_rows_per_src=[16276, 16384, 16372, 16382, 16344, 13976, 16350, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=838795264 dispatch_meta_bytes=457200 combine_payload_bytes=838795264 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=839252464 ingress_bytes=1719599104 link_bytes=1719599104 token_bytes=1678047728 weight_movement_bytes=880803840 total_nvlink_bytes=2558851568 weight_to_token_ratio=0.524898 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.629037186e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=872054784 dispatch_meta_bytes=458816 combine_payload_bytes=872054784 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=872513600 ingress_bytes=1929019392 link_bytes=1929019392 token_bytes=1744568384 weight_movement_bytes=1056964608 total_nvlink_bytes=2801532992 weight_to_token_ratio=0.605860 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.080170338e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124110 payload_rows_remote=107910 dup_rows=6962 dup_rows_remote=2514 dedup_ratio=0.053116 router_tokens_per_expert=[8022, 13, 8175, 0, 8192, 0, 1713, 0, 0, 0, 0, 0, 1, 5, 33, 26, 0, 8187, 0, 0, 8192, 0, 0, 6632, 0, 0, 8190, 8155, 0, 0, 0, 0] payload_rows_per_dst=[16200, 12848, 16350, 16384, 16374, 13264, 16310, 16380] dup_rows_per_dst=[4448, 2384, 0, 130, 0, 0, 0, 0] combine_rows_per_src=[16200, 12848, 16350, 16384, 16374, 13264, 16310, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=888373248 dispatch_meta_bytes=458752 combine_payload_bytes=888373248 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=888832000 ingress_bytes=1240694784 link_bytes=1240694784 token_bytes=1777205248 weight_movement_bytes=352321536 total_nvlink_bytes=2129526784 weight_to_token_ratio=0.198245 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.947372899e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=921190400 dispatch_meta_bytes=449952 combine_payload_bytes=921190400 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=921640352 ingress_bytes=2330476544 link_bytes=2330476544 token_bytes=1842830752 weight_movement_bytes=1409286144 total_nvlink_bytes=3252116896 weight_to_token_ratio=0.764740 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.304969586e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126934 payload_rows_remote=113440 dup_rows=4138 dup_rows_remote=4138 dedup_ratio=0.031570 router_tokens_per_expert=[7691, 20, 8173, 5, 8192, 0, 2013, 0, 0, 0, 0, 0, 0, 7, 35, 9, 0, 8103, 0, 0, 8192, 0, 0, 6747, 0, 0, 8187, 8162, 0, 0, 0, 0] payload_rows_per_dst=[15422, 16384, 16346, 16384, 16206, 13494, 16324, 16374] dup_rows_per_dst=[10, 4026, 0, 102, 0, 0, 0, 0] combine_rows_per_src=[15422, 16384, 16346, 16384, 16206, 13494, 16324, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=918274048 dispatch_meta_bytes=458728 combine_payload_bytes=918274048 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=918732776 ingress_bytes=1622917120 link_bytes=1622917120 token_bytes=1837006824 weight_movement_bytes=704643072 total_nvlink_bytes=2541649896 weight_to_token_ratio=0.383582 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.087658433e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=883998720 dispatch_meta_bytes=441696 combine_payload_bytes=883998720 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=884440416 ingress_bytes=1940963328 link_bytes=1940963328 token_bytes=1768439136 weight_movement_bytes=1056964608 total_nvlink_bytes=2825403744 weight_to_token_ratio=0.597682 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.086858444e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=929300480 dispatch_meta_bytes=470312 combine_payload_bytes=929300480 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=929770792 ingress_bytes=1633943552 link_bytes=1633943552 token_bytes=1859071272 weight_movement_bytes=704643072 total_nvlink_bytes=2563714344 weight_to_token_ratio=0.379030 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.026515 gemm_active_groups=8 gemm_peak_group_rows=27904 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.149401850e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119918 payload_rows_remote=103534 dup_rows=11154 dup_rows_remote=11128 dedup_ratio=0.085098 router_tokens_per_expert=[4, 3178, 0, 8186, 75, 6, 6, 2, 0, 2, 8179, 5, 7, 7641, 6, 8186, 8187, 0, 8177, 0, 5, 6, 8178, 3, 8, 0, 0, 10, 0, 5475, 4, 0] payload_rows_per_dst=[6364, 16384, 16370, 15308, 16354, 16374, 16384, 16380] dup_rows_per_dst=[0, 166, 2, 0, 0, 10, 26, 10950] combine_rows_per_src=[6364, 16384, 16370, 15308, 16354, 16374, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117508 payload_rows_remote=101124 dup_rows=13564 dup_rows_remote=13520 dedup_ratio=0.103485 router_tokens_per_expert=[7581, 3, 8182, 0, 8192, 2, 2405, 0, 0, 23, 0, 0, 0, 0, 48, 0, 0, 8182, 0, 0, 8192, 0, 0, 6368, 0, 0, 8192, 8166, 0, 0, 0, 0] payload_rows_per_dst=[15090, 4810, 16374, 15770, 16364, 16384, 16332, 16384] dup_rows_per_dst=[2, 4, 112, 96, 0, 12992, 314, 44] combine_rows_per_src=[15090, 4810, 16374, 15770, 16364, 16384, 16332, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127452 payload_rows_remote=111078 dup_rows=3620 dup_rows_remote=3620 dedup_ratio=0.027618 router_tokens_per_expert=[7966, 5, 8186, 0, 8192, 0, 1779, 0, 0, 0, 0, 0, 0, 3, 19, 9, 0, 8187, 0, 0, 8192, 0, 0, 6632, 0, 0, 8192, 8174, 0, 0, 0, 0] payload_rows_per_dst=[15942, 16384, 16372, 16384, 16374, 13264, 16348, 16384] dup_rows_per_dst=[0, 3558, 0, 62, 0, 0, 0, 0] combine_rows_per_src=[15942, 16384, 16372, 16384, 16374, 13264, 16348, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129526 payload_rows_remote=113142 dup_rows=1546 dup_rows_remote=1462 dedup_ratio=0.011795 router_tokens_per_expert=[7986, 10, 8169, 0, 8192, 0, 731, 0, 0, 0, 0, 0, 1, 4, 24, 13, 0, 8187, 0, 0, 8192, 0, 0, 7651, 0, 0, 8192, 8184, 0, 0, 0, 0] payload_rows_per_dst=[15992, 16384, 16338, 16384, 16374, 15302, 16368, 16384] dup_rows_per_dst=[0, 1462, 0, 84, 0, 0, 0, 0] combine_rows_per_src=[15992, 16384, 16338, 16384, 16374, 15302, 16368, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125118 payload_rows_remote=108734 dup_rows=5954 dup_rows_remote=5740 dedup_ratio=0.045425 router_tokens_per_expert=[0, 2869, 0, 8192, 107, 0, 0, 0, 0, 1, 8192, 0, 0, 8124, 0, 8192, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5283, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16248, 16384, 16384, 16384, 10566] dup_rows_per_dst=[5738, 214, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16248, 16384, 16384, 16384, 10566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122916 payload_rows_remote=106532 dup_rows=8156 dup_rows_remote=8152 dedup_ratio=0.062225 router_tokens_per_expert=[0, 4016, 0, 8190, 51, 1, 0, 0, 0, 2, 8191, 1, 5, 7861, 1, 8191, 8191, 0, 8190, 1, 1, 1, 8186, 0, 2, 0, 0, 1, 1, 4451, 1, 0] payload_rows_per_dst=[16380, 16384, 16384, 15724, 16382, 16374, 16384, 8904] dup_rows_per_dst=[8032, 102, 4, 10, 2, 2, 2, 2] combine_rows_per_src=[16380, 16384, 16384, 15724, 16382, 16374, 16384, 8904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=848150528 dispatch_meta_bytes=458648 combine_payload_bytes=848150528 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=848609176 ingress_bytes=1200472064 link_bytes=1200472064 token_bytes=1696759704 weight_movement_bytes=352321536 total_nvlink_bytes=2049081240 weight_to_token_ratio=0.207644 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.722142457e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=828407808 dispatch_meta_bytes=458576 combine_payload_bytes=828407808 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=828866384 ingress_bytes=1533050880 link_bytes=1533050880 token_bytes=1657274192 weight_movement_bytes=704643072 total_nvlink_bytes=2361917264 weight_to_token_ratio=0.425182 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.584445001e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=909950976 dispatch_meta_bytes=458792 combine_payload_bytes=909950976 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=910409768 ingress_bytes=1262272512 link_bytes=1262272512 token_bytes=1820360744 weight_movement_bytes=352321536 total_nvlink_bytes=2172682280 weight_to_token_ratio=0.193545 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.068199169e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=926859264 dispatch_meta_bytes=458416 combine_payload_bytes=926859264 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=927317680 ingress_bytes=1279180800 link_bytes=1279180800 token_bytes=1854176944 weight_movement_bytes=352321536 total_nvlink_bytes=2206498480 weight_to_token_ratio=0.190015 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.162878524e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123526 payload_rows_remote=107142 dup_rows=7546 dup_rows_remote=248 dedup_ratio=0.057571 router_tokens_per_expert=[0, 3469, 0, 8192, 111, 0, 0, 0, 0, 0, 8191, 0, 7, 7993, 0, 8192, 8189, 0, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 4812, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 15636, 16380, 16376, 16360, 9624] dup_rows_per_dst=[7298, 222, 0, 4, 22, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 15636, 16380, 16376, 16360, 9624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=890748928 dispatch_meta_bytes=457896 combine_payload_bytes=890748928 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=891206824 ingress_bytes=1243070464 link_bytes=1243070464 token_bytes=1781955752 weight_movement_bytes=352321536 total_nvlink_bytes=2134277288 weight_to_token_ratio=0.197716 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.960675715e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=872710144 dispatch_meta_bytes=458736 combine_payload_bytes=872710144 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=873168880 ingress_bytes=1225031680 link_bytes=1225031680 token_bytes=1745879024 weight_movement_bytes=352321536 total_nvlink_bytes=2098200560 weight_to_token_ratio=0.201802 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.859666055e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119542 payload_rows_remote=103170 dup_rows=11530 dup_rows_remote=11530 dedup_ratio=0.087967 router_tokens_per_expert=[0, 2477, 0, 8192, 352, 0, 0, 0, 0, 0, 8192, 0, 17, 8159, 0, 8192, 8181, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 5396, 0, 0] payload_rows_per_dst=[4954, 16384, 16384, 16318, 16366, 16372, 16380, 16384] dup_rows_per_dst=[0, 704, 0, 34, 0, 0, 0, 10792] combine_rows_per_src=[4954, 16384, 16384, 16318, 16366, 16372, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=877707264 dispatch_meta_bytes=429560 combine_payload_bytes=877707264 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=878136824 ingress_bytes=2110832640 link_bytes=2110832640 token_bytes=1755844088 weight_movement_bytes=1233125376 total_nvlink_bytes=2988969464 weight_to_token_ratio=0.702298 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=19328 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.181978168e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=845168640 dispatch_meta_bytes=458800 combine_payload_bytes=845168640 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=845627440 ingress_bytes=1197490176 link_bytes=1197490176 token_bytes=1690796080 weight_movement_bytes=352321536 total_nvlink_bytes=2043117616 weight_to_token_ratio=0.208376 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.705445129e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130912 payload_rows_remote=114528 dup_rows=160 dup_rows_remote=156 dedup_ratio=0.001221 router_tokens_per_expert=[7, 6, 8, 8178, 8183, 12, 0, 5, 8185, 8134, 6, 8187, 0, 0, 55, 0, 8185, 0, 5, 8183, 8178, 0, 0, 6, 0, 0, 1, 6, 0, 0, 0, 6] payload_rows_per_dst=[16376, 16376, 16280, 16372, 16376, 16368, 16384, 16380] dup_rows_per_dst=[22, 24, 0, 110, 0, 0, 4, 0] combine_rows_per_src=[16376, 16376, 16280, 16372, 16376, 16368, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121072 payload_rows_remote=104688 dup_rows=10000 dup_rows_remote=3040 dedup_ratio=0.076294 router_tokens_per_expert=[0, 4541, 0, 8192, 200, 0, 0, 0, 0, 1, 8192, 0, 0, 8162, 0, 8192, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3480, 0, 0] payload_rows_per_dst=[9082, 13746, 16380, 16328, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 400, 2, 2594, 30, 14, 0, 6960] combine_rows_per_src=[9082, 13746, 16380, 16328, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124182 payload_rows_remote=108018 dup_rows=6890 dup_rows_remote=6880 dedup_ratio=0.052567 router_tokens_per_expert=[0, 3222, 0, 8192, 218, 0, 0, 0, 0, 0, 8192, 0, 5, 8082, 0, 8192, 8190, 0, 8192, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 4862, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16164, 16380, 16378, 16384, 9724] dup_rows_per_dst=[6444, 436, 0, 10, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16164, 16380, 16378, 16384, 9724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122372 payload_rows_remote=105992 dup_rows=8700 dup_rows_remote=8692 dedup_ratio=0.066376 router_tokens_per_expert=[0, 3726, 0, 8192, 197, 0, 0, 0, 0, 0, 8192, 0, 4, 8161, 0, 8192, 8190, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 4300, 0, 0] payload_rows_per_dst=[12966, 16384, 16384, 16324, 16380, 16380, 16384, 11170] dup_rows_per_dst=[4662, 394, 10, 370, 8, 24, 0, 3232] combine_rows_per_src=[12966, 16384, 16384, 16324, 16380, 16380, 16384, 11170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130800 payload_rows_remote=114420 dup_rows=272 dup_rows_remote=272 dedup_ratio=0.002075 router_tokens_per_expert=[0, 0, 0, 8192, 8190, 0, 0, 0, 8191, 8058, 0, 8192, 0, 0, 136, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16116, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 270, 0, 0, 2, 0] combine_rows_per_src=[16384, 16380, 16116, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130970 payload_rows_remote=114658 dup_rows=102 dup_rows_remote=102 dedup_ratio=0.000778 router_tokens_per_expert=[1, 1, 4, 8186, 8187, 5, 0, 1, 8186, 8155, 1, 8182, 0, 0, 47, 4, 8191, 0, 1, 8191, 8188, 0, 0, 1, 0, 0, 0, 1, 0, 0, 2, 1] payload_rows_per_dst=[16374, 16384, 16312, 16382, 16384, 16378, 16382, 16374] dup_rows_per_dst=[10, 2, 0, 86, 0, 0, 0, 4] combine_rows_per_src=[16374, 16384, 16312, 16382, 16384, 16378, 16382, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938213376 dispatch_meta_bytes=458736 combine_payload_bytes=938213376 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938672112 ingress_bytes=1290534912 link_bytes=1290534912 token_bytes=1876885488 weight_movement_bytes=352321536 total_nvlink_bytes=2229207024 weight_to_token_ratio=0.187716 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226456811e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=857604096 dispatch_meta_bytes=430912 combine_payload_bytes=857604096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=858035008 ingress_bytes=1209925632 link_bytes=1209925632 token_bytes=1715639104 weight_movement_bytes=352321536 total_nvlink_bytes=2067960640 weight_to_token_ratio=0.205359 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.775078491e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=884883456 dispatch_meta_bytes=459592 combine_payload_bytes=884883456 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=885343048 ingress_bytes=2118008832 link_bytes=2118008832 token_bytes=1770226504 weight_movement_bytes=1233125376 total_nvlink_bytes=3003351880 weight_to_token_ratio=0.696592 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.185996536e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 1, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=868286464 dispatch_meta_bytes=458736 combine_payload_bytes=868286464 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=868745200 ingress_bytes=1572929536 link_bytes=1572929536 token_bytes=1737031664 weight_movement_bytes=704643072 total_nvlink_bytes=2441674736 weight_to_token_ratio=0.405659 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.807748829e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129358 payload_rows_remote=112976 dup_rows=1714 dup_rows_remote=1706 dedup_ratio=0.013077 router_tokens_per_expert=[0, 0, 0, 8188, 8188, 1, 0, 0, 8192, 8094, 0, 8192, 0, 0, 94, 7, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0] payload_rows_per_dst=[16382, 16382, 16202, 14878, 16384, 16384, 16370, 16376] dup_rows_per_dst=[8, 14, 1498, 182, 12, 0, 0, 0] combine_rows_per_src=[16382, 16382, 16202, 14878, 16384, 16384, 16370, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130792 payload_rows_remote=114408 dup_rows=280 dup_rows_remote=280 dedup_ratio=0.002136 router_tokens_per_expert=[0, 0, 0, 8190, 8175, 8, 0, 0, 8192, 8064, 0, 8190, 0, 0, 126, 11, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0] payload_rows_per_dst=[16380, 16366, 16128, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 272, 0, 0, 2, 6] combine_rows_per_src=[16380, 16366, 16128, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937328640 dispatch_meta_bytes=458768 combine_payload_bytes=937328640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=937787408 ingress_bytes=1113489408 link_bytes=1113489408 token_bytes=1875116048 weight_movement_bytes=176160768 total_nvlink_bytes=2051276816 weight_to_token_ratio=0.093947 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.235075891e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=459040 combine_payload_bytes=939278336 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=939737376 ingress_bytes=2348564480 link_bytes=2348564480 token_bytes=1879015712 weight_movement_bytes=1409286144 total_nvlink_bytes=3288301856 weight_to_token_ratio=0.750013 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315098075e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=925499392 dispatch_meta_bytes=458728 combine_payload_bytes=925499392 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=925958120 ingress_bytes=1277820928 link_bytes=1277820928 token_bytes=1851457512 weight_movement_bytes=352321536 total_nvlink_bytes=2203779048 weight_to_token_ratio=0.190294 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.155263808e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937230336 dispatch_meta_bytes=458752 combine_payload_bytes=937230336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937689088 ingress_bytes=1289551872 link_bytes=1289551872 token_bytes=1874919424 weight_movement_bytes=352321536 total_nvlink_bytes=2227240960 weight_to_token_ratio=0.187913 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.220952198e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130460 payload_rows_remote=114096 dup_rows=612 dup_rows_remote=408 dedup_ratio=0.004669 router_tokens_per_expert=[2, 0, 7966, 3, 8178, 0, 5, 6, 8186, 8177, 8185, 8, 6, 6, 0, 0, 0, 0, 7, 183, 8187, 4, 5, 0, 247, 16, 0, 0, 4, 8021, 0, 8134] payload_rows_per_dst=[15940, 16368, 16370, 16384, 16374, 16384, 16364, 16276] dup_rows_per_dst=[2, 10, 0, 12, 376, 8, 204, 0] combine_rows_per_src=[15940, 16368, 16370, 16384, 16374, 16384, 16364, 16276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130838 payload_rows_remote=114454 dup_rows=234 dup_rows_remote=222 dedup_ratio=0.001785 router_tokens_per_expert=[0, 0, 0, 8192, 8191, 0, 0, 0, 8192, 8094, 0, 8192, 0, 0, 98, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16158, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 196, 2, 22, 2, 12] combine_rows_per_src=[16384, 16382, 16158, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130848 payload_rows_remote=114464 dup_rows=224 dup_rows_remote=6 dedup_ratio=0.001709 router_tokens_per_expert=[0, 0, 0, 8188, 8185, 4, 0, 0, 8192, 8087, 0, 8192, 0, 0, 105, 4, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16378, 16174, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 218, 0, 0, 6, 0] combine_rows_per_src=[16376, 16378, 16174, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130884 payload_rows_remote=114500 dup_rows=188 dup_rows_remote=188 dedup_ratio=0.001434 router_tokens_per_expert=[0, 0, 0, 8188, 8188, 4, 0, 0, 8192, 8102, 0, 8192, 0, 0, 90, 4, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16384, 16204, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 188, 0, 0, 0, 0] combine_rows_per_src=[16376, 16384, 16204, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130748 payload_rows_remote=114364 dup_rows=324 dup_rows_remote=324 dedup_ratio=0.002472 router_tokens_per_expert=[0, 0, 8109, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 162, 0, 0, 0, 0, 8192, 0, 8113] payload_rows_per_dst=[16218, 16384, 16384, 16384, 16384, 16384, 16384, 16226] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 324, 0] combine_rows_per_src=[16218, 16384, 16384, 16384, 16384, 16384, 16384, 16226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=934674432 dispatch_meta_bytes=458016 combine_payload_bytes=934674432 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935132448 ingress_bytes=1286995968 link_bytes=1286995968 token_bytes=1869806880 weight_movement_bytes=352321536 total_nvlink_bytes=2222128416 weight_to_token_ratio=0.188427 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.206640202e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130704 payload_rows_remote=114322 dup_rows=368 dup_rows_remote=368 dedup_ratio=0.002808 router_tokens_per_expert=[0, 2, 8025, 0, 8190, 0, 1, 1, 8191, 8190, 8191, 1, 1, 1, 0, 0, 0, 0, 0, 79, 8190, 1, 1, 0, 148, 4, 0, 0, 0, 8139, 1, 8179] payload_rows_per_dst=[16050, 16382, 16382, 16384, 16382, 16382, 16382, 16360] dup_rows_per_dst=[4, 2, 0, 2, 158, 2, 200, 0] combine_rows_per_src=[16050, 16382, 16382, 16384, 16382, 16382, 16382, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128946 payload_rows_remote=112642 dup_rows=2126 dup_rows_remote=844 dedup_ratio=0.016220 router_tokens_per_expert=[0, 0, 8150, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 150, 0, 0, 0, 0, 8192, 0, 8084] payload_rows_per_dst=[16304, 16384, 16384, 16370, 15862, 16376, 16384, 14882] dup_rows_per_dst=[1282, 6, 538, 0, 0, 0, 300, 0] combine_rows_per_src=[16304, 16384, 16384, 16370, 15862, 16376, 16384, 14882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937607168 dispatch_meta_bytes=458704 combine_payload_bytes=937607168 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=938065872 ingress_bytes=1466089472 link_bytes=1466089472 token_bytes=1875673040 weight_movement_bytes=528482304 total_nvlink_bytes=2404155344 weight_to_token_ratio=0.281756 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.209489067e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937689088 dispatch_meta_bytes=457880 combine_payload_bytes=937689088 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938146968 ingress_bytes=1290010624 link_bytes=1290010624 token_bytes=1875836056 weight_movement_bytes=352321536 total_nvlink_bytes=2228157592 weight_to_token_ratio=0.187821 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.026515 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223521017e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937984000 dispatch_meta_bytes=458752 combine_payload_bytes=937984000 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=938442752 ingress_bytes=1994948608 link_bytes=1994948608 token_bytes=1876426752 weight_movement_bytes=1056964608 total_nvlink_bytes=2933391360 weight_to_token_ratio=0.563286 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117087947e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936869888 dispatch_meta_bytes=458752 combine_payload_bytes=936869888 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=937328640 ingress_bytes=1113030656 link_bytes=1113030656 token_bytes=1874198528 weight_movement_bytes=176160768 total_nvlink_bytes=2050359296 weight_to_token_ratio=0.093993 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.232507072e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130902 payload_rows_remote=114518 dup_rows=170 dup_rows_remote=170 dedup_ratio=0.001297 router_tokens_per_expert=[0, 0, 8152, 0, 8192, 0, 0, 0, 8192, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 85, 0, 0, 0, 0, 8192, 0, 8148] payload_rows_per_dst=[16304, 16384, 16382, 16384, 16384, 16384, 16384, 16296] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 170, 0] combine_rows_per_src=[16304, 16384, 16382, 16384, 16384, 16384, 16384, 16296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936525824 dispatch_meta_bytes=458760 combine_payload_bytes=936525824 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=936984584 ingress_bytes=2169651200 link_bytes=2169651200 token_bytes=1873510408 weight_movement_bytes=1233125376 total_nvlink_bytes=3106635784 weight_to_token_ratio=0.658190 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.214914106e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=922763264 dispatch_meta_bytes=453944 combine_payload_bytes=922763264 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=923217208 ingress_bytes=1275084800 link_bytes=1275084800 token_bytes=1845980472 weight_movement_bytes=352321536 total_nvlink_bytes=2198302008 weight_to_token_ratio=0.190859 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.139942633e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938131456 dispatch_meta_bytes=458752 combine_payload_bytes=938131456 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=938590208 ingress_bytes=1290452992 link_bytes=1290452992 token_bytes=1876721664 weight_movement_bytes=352321536 total_nvlink_bytes=2229043200 weight_to_token_ratio=0.187732 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.225998094e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129684 payload_rows_remote=113528 dup_rows=1388 dup_rows_remote=1388 dedup_ratio=0.010590 router_tokens_per_expert=[5, 0, 100, 7, 0, 0, 2, 6, 0, 8187, 0, 6, 4, 4, 6, 8166, 11, 14, 5, 559, 26, 8175, 8179, 0, 0, 0, 8167, 8078, 4, 49, 7591, 8185] payload_rows_per_dst=[16384, 16376, 16382, 16344, 16384, 16370, 16156, 15288] dup_rows_per_dst=[174, 0, 4, 16, 1164, 30, 0, 0] combine_rows_per_src=[16384, 16376, 16382, 16344, 16384, 16370, 16156, 15288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130746 payload_rows_remote=114362 dup_rows=326 dup_rows_remote=326 dedup_ratio=0.002487 router_tokens_per_expert=[0, 0, 8127, 0, 8192, 0, 0, 0, 8192, 8187, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 163, 0, 0, 0, 0, 8192, 0, 8099] payload_rows_per_dst=[16254, 16384, 16374, 16384, 16384, 16384, 16384, 16198] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 326, 0] combine_rows_per_src=[16254, 16384, 16374, 16384, 16384, 16384, 16384, 16198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128708 payload_rows_remote=112422 dup_rows=2364 dup_rows_remote=116 dedup_ratio=0.018036 router_tokens_per_expert=[0, 0, 8183, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 73, 0, 0, 0, 0, 8192, 0, 8128] payload_rows_per_dst=[16366, 16384, 15878, 16384, 16384, 16384, 14642, 16286] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 116, 2248] combine_rows_per_src=[16366, 16384, 15878, 16384, 16384, 16384, 14642, 16286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129576 payload_rows_remote=113192 dup_rows=1496 dup_rows_remote=1496 dedup_ratio=0.011414 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 8168, 1, 3, 0, 721, 0, 8192, 8192, 0, 0, 0, 8192, 8181, 0, 74, 7464, 8192] payload_rows_per_dst=[16384, 16384, 16312, 16336, 16384, 16384, 16362, 15030] dup_rows_per_dst=[0, 0, 0, 0, 1450, 0, 0, 46] combine_rows_per_src=[16384, 16384, 16312, 16336, 16384, 16384, 16362, 15030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130948 payload_rows_remote=114564 dup_rows=124 dup_rows_remote=124 dedup_ratio=0.000946 router_tokens_per_expert=[0, 0, 8174, 0, 8192, 0, 0, 0, 8192, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 65, 0, 0, 0, 0, 8189, 0, 8150] payload_rows_per_dst=[16348, 16384, 16380, 16384, 16384, 16384, 16384, 16300] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 124, 0] combine_rows_per_src=[16348, 16384, 16380, 16384, 16384, 16384, 16384, 16300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129674 payload_rows_remote=113300 dup_rows=1398 dup_rows_remote=1398 dedup_ratio=0.010666 router_tokens_per_expert=[0, 0, 10, 1, 0, 0, 0, 1, 0, 8184, 2, 1, 1, 1, 1, 8160, 2, 7, 1, 676, 9, 8182, 8182, 0, 1, 0, 8190, 8186, 1, 76, 7470, 8191] payload_rows_per_dst=[16384, 16366, 16374, 16322, 16384, 16378, 16372, 15094] dup_rows_per_dst=[18, 0, 0, 4, 1370, 4, 2, 0] combine_rows_per_src=[16384, 16366, 16374, 16322, 16384, 16378, 16372, 15094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=930021376 dispatch_meta_bytes=459664 combine_payload_bytes=930021376 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=930481040 ingress_bytes=1634664448 link_bytes=1634664448 token_bytes=1860502416 weight_movement_bytes=704643072 total_nvlink_bytes=2565145488 weight_to_token_ratio=0.378738 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.153438566e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128260 payload_rows_remote=111878 dup_rows=2812 dup_rows_remote=2804 dedup_ratio=0.021454 router_tokens_per_expert=[0, 0, 11, 0, 1, 0, 0, 0, 0, 8160, 1, 0, 0, 0, 0, 8174, 5, 3, 0, 716, 0, 8192, 8191, 0, 0, 0, 8184, 8128, 0, 144, 7434, 8192] payload_rows_per_dst=[16382, 16384, 16378, 16348, 16384, 16362, 16256, 13766] dup_rows_per_dst=[8, 22, 478, 324, 1448, 18, 514, 0] combine_rows_per_src=[16382, 16384, 16378, 16348, 16384, 16362, 16256, 13766] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936853504 dispatch_meta_bytes=458752 combine_payload_bytes=936853504 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937312256 ingress_bytes=1289175040 link_bytes=1289175040 token_bytes=1874165760 weight_movement_bytes=352321536 total_nvlink_bytes=2226487296 weight_to_token_ratio=0.187988 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.218842096e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=920961024 dispatch_meta_bytes=450152 combine_payload_bytes=920961024 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=921411176 ingress_bytes=1977925632 link_bytes=1977925632 token_bytes=1842372200 weight_movement_bytes=1056964608 total_nvlink_bytes=2899336808 weight_to_token_ratio=0.573698 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.107555791e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=927268864 dispatch_meta_bytes=458752 combine_payload_bytes=927268864 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=927727616 ingress_bytes=1631911936 link_bytes=1631911936 token_bytes=1854996480 weight_movement_bytes=704643072 total_nvlink_bytes=2559639552 weight_to_token_ratio=0.379862 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.138025648e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129106 payload_rows_remote=112722 dup_rows=1966 dup_rows_remote=1966 dedup_ratio=0.014999 router_tokens_per_expert=[0, 0, 20, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 8148, 7, 13, 0, 909, 0, 8192, 8192, 0, 0, 0, 8187, 8132, 0, 283, 7134, 8192] payload_rows_per_dst=[16384, 16384, 16254, 16296, 16384, 16384, 16264, 14756] dup_rows_per_dst=[30, 0, 0, 0, 1858, 0, 0, 78] combine_rows_per_src=[16384, 16384, 16254, 16296, 16384, 16384, 16264, 14756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938508288 dispatch_meta_bytes=458752 combine_payload_bytes=938508288 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938967040 ingress_bytes=1290829824 link_bytes=1290829824 token_bytes=1877475328 weight_movement_bytes=352321536 total_nvlink_bytes=2229796864 weight_to_token_ratio=0.187657 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228108196e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=928153600 dispatch_meta_bytes=458792 combine_payload_bytes=928153600 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928612392 ingress_bytes=1280475136 link_bytes=1280475136 token_bytes=1856765992 weight_movement_bytes=352321536 total_nvlink_bytes=2209087528 weight_to_token_ratio=0.189750 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.170126265e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=916504576 dispatch_meta_bytes=458728 combine_payload_bytes=916504576 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=916963304 ingress_bytes=1268826112 link_bytes=1268826112 token_bytes=1833467880 weight_movement_bytes=352321536 total_nvlink_bytes=2185789416 weight_to_token_ratio=0.192161 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.104896593e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=923418624 dispatch_meta_bytes=458752 combine_payload_bytes=923418624 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=923877376 ingress_bytes=1980383232 link_bytes=1980383232 token_bytes=1847296000 weight_movement_bytes=1056964608 total_nvlink_bytes=2904260608 weight_to_token_ratio=0.572169 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.108931945e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130846 payload_rows_remote=114538 dup_rows=226 dup_rows_remote=222 dedup_ratio=0.001724 router_tokens_per_expert=[9, 5, 8165, 8120, 4, 0, 1, 0, 0, 0, 6, 6, 0, 6, 9, 97, 8182, 8, 0, 8185, 0, 8172, 25, 0, 8186, 8, 8147, 1, 0, 0, 7, 8187] payload_rows_per_dst=[16260, 16380, 16384, 16384, 16380, 16366, 16308, 16384] dup_rows_per_dst=[8, 0, 12, 170, 0, 28, 4, 4] combine_rows_per_src=[16260, 16380, 16384, 16384, 16380, 16366, 16308, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128166 payload_rows_remote=111804 dup_rows=2906 dup_rows_remote=2906 dedup_ratio=0.022171 router_tokens_per_expert=[0, 0, 7, 0, 0, 0, 0, 0, 0, 8140, 3, 0, 0, 0, 0, 8181, 12, 2, 0, 1421, 0, 8192, 8192, 0, 0, 0, 8192, 8137, 0, 88, 6777, 8192] payload_rows_per_dst=[16384, 16384, 16286, 16362, 16384, 16384, 16274, 13708] dup_rows_per_dst=[14, 0, 0, 0, 2870, 0, 0, 22] combine_rows_per_src=[16384, 16384, 16286, 16362, 16384, 16384, 16274, 13708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117314 payload_rows_remote=101676 dup_rows=13758 dup_rows_remote=1148 dedup_ratio=0.104965 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 8188, 0, 2, 0, 1011, 0, 8192, 8189, 0, 0, 0, 8192, 8141, 0, 152, 7141, 8192] payload_rows_per_dst=[16132, 16380, 16270, 16376, 3862, 16374, 16282, 15638] dup_rows_per_dst=[2, 0, 0, 0, 894, 0, 252, 12610] combine_rows_per_src=[16132, 16380, 16270, 16376, 3862, 16374, 16282, 15638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 8190, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130984 payload_rows_remote=114600 dup_rows=88 dup_rows_remote=86 dedup_ratio=0.000671 router_tokens_per_expert=[0, 1, 8189, 8187, 1, 0, 0, 0, 0, 0, 1, 1, 0, 2, 1, 38, 8191, 1, 1, 8191, 0, 8158, 1, 0, 8191, 4, 8191, 1, 0, 0, 0, 8185] payload_rows_per_dst=[16376, 16384, 16384, 16384, 16384, 16318, 16384, 16370] dup_rows_per_dst=[0, 0, 2, 76, 2, 0, 8, 0] combine_rows_per_src=[16376, 16384, 16384, 16384, 16384, 16318, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938295296 dispatch_meta_bytes=459040 combine_payload_bytes=938295296 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=938754336 ingress_bytes=1819099136 link_bytes=1819099136 token_bytes=1877049632 weight_movement_bytes=880803840 total_nvlink_bytes=2757853472 weight_to_token_ratio=0.469249 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018619583e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 2, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130728 payload_rows_remote=114344 dup_rows=344 dup_rows_remote=8 dedup_ratio=0.002625 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8184] payload_rows_per_dst=[16384, 16384, 16384, 16074, 16384, 16376, 16374, 16368] dup_rows_per_dst=[336, 4, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16074, 16384, 16376, 16374, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=915898368 dispatch_meta_bytes=458840 combine_payload_bytes=915898368 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=916357208 ingress_bytes=1268219904 link_bytes=1268219904 token_bytes=1832255576 weight_movement_bytes=352321536 total_nvlink_bytes=2184577112 weight_to_token_ratio=0.192288 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.101502081e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=832929792 dispatch_meta_bytes=411296 combine_payload_bytes=832929792 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=833341088 ingress_bytes=2594537472 link_bytes=2594537472 token_bytes=1666270880 weight_movement_bytes=1761607680 total_nvlink_bytes=3427878560 weight_to_token_ratio=1.057216 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=29312 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.452832683e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129570 payload_rows_remote=113186 dup_rows=1502 dup_rows_remote=34 dedup_ratio=0.011459 router_tokens_per_expert=[0, 0, 10, 0, 4, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 8172, 4, 1, 0, 729, 0, 8192, 8192, 0, 0, 0, 8186, 8169, 0, 97, 7421, 8192] payload_rows_per_dst=[16384, 16384, 16334, 16344, 16384, 16384, 16338, 15018] dup_rows_per_dst=[8, 8, 0, 0, 1468, 0, 0, 18] combine_rows_per_src=[16384, 16384, 16334, 16344, 16384, 16384, 16338, 15018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879441408 weight_movement_bytes=528482304 total_nvlink_bytes=2407923712 weight_to_token_ratio=0.281191 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131040 payload_rows_remote=114656 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8176] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16352] dup_rows_per_dst=[0, 0, 0, 32, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938803200 dispatch_meta_bytes=458744 combine_payload_bytes=938803200 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939261944 ingress_bytes=1291124736 link_bytes=1291124736 token_bytes=1878065144 weight_movement_bytes=352321536 total_nvlink_bytes=2230386680 weight_to_token_ratio=0.187598 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229759580e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936706048 dispatch_meta_bytes=457408 combine_payload_bytes=936706048 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=937163456 ingress_bytes=2169831424 link_bytes=2169831424 token_bytes=1873869504 weight_movement_bytes=1233125376 total_nvlink_bytes=3106994880 weight_to_token_ratio=0.658064 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.215015024e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=927219712 dispatch_meta_bytes=452880 combine_payload_bytes=927219712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=927672592 ingress_bytes=1279541248 link_bytes=1279541248 token_bytes=1854892304 weight_movement_bytes=352321536 total_nvlink_bytes=2207213840 weight_to_token_ratio=0.189942 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=29824 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.164896882e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130834 payload_rows_remote=114450 dup_rows=238 dup_rows_remote=216 dedup_ratio=0.001816 router_tokens_per_expert=[0, 0, 7, 0, 0, 8120, 8184, 0, 7, 4, 8185, 8149, 8184, 11, 0, 8185, 6, 0, 6, 92, 4, 0, 8181, 7, 0, 7, 4, 7, 0, 1, 0, 8185] payload_rows_per_dst=[16382, 16240, 16312, 16384, 16384, 16376, 16384, 16372] dup_rows_per_dst=[2, 0, 8, 6, 192, 8, 22, 0] combine_rows_per_src=[16382, 16240, 16312, 16384, 16384, 16376, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=458752 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939720704 ingress_bytes=1291583488 link_bytes=1291583488 token_bytes=1878982656 weight_movement_bytes=352321536 total_nvlink_bytes=2231304192 weight_to_token_ratio=0.187507 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232328399e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114674 dup_rows=14 dup_rows_remote=0 dedup_ratio=0.000107 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8185] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16370] dup_rows_per_dst=[0, 0, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130840 payload_rows_remote=114456 dup_rows=232 dup_rows_remote=150 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 8191, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16172, 16384, 16370, 16384, 16382, 16382, 16382, 16384] dup_rows_per_dst=[0, 0, 0, 4, 0, 70, 76, 82] combine_rows_per_src=[16172, 16384, 16370, 16384, 16382, 16382, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=937574400 dispatch_meta_bytes=458664 combine_payload_bytes=937574400 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938033064 ingress_bytes=1289895936 link_bytes=1289895936 token_bytes=1875607464 weight_movement_bytes=352321536 total_nvlink_bytes=2227929000 weight_to_token_ratio=0.187844 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.222878813e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130990 payload_rows_remote=114642 dup_rows=82 dup_rows_remote=80 dedup_ratio=0.000626 router_tokens_per_expert=[2, 0, 1, 0, 0, 8176, 8187, 0, 1, 1, 8191, 8173, 8191, 2, 3, 8191, 1, 0, 1, 31, 1, 0, 8190, 1, 0, 1, 1, 0, 0, 0, 0, 8190] payload_rows_per_dst=[16384, 16352, 16348, 16384, 16376, 16382, 16384, 16380] dup_rows_per_dst=[4, 0, 2, 8, 64, 2, 2, 0] combine_rows_per_src=[16384, 16352, 16348, 16384, 16376, 16382, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130762 payload_rows_remote=114380 dup_rows=310 dup_rows_remote=310 dedup_ratio=0.002365 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 8192, 8192, 8192, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16382, 16384, 16384, 16332, 16152, 16384, 16360, 16384] dup_rows_per_dst=[0, 176, 134, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16332, 16152, 16384, 16360, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=458696 combine_payload_bytes=939409408 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939868104 ingress_bytes=1291730944 link_bytes=1291730944 token_bytes=1879277512 weight_movement_bytes=352321536 total_nvlink_bytes=2231599048 weight_to_token_ratio=0.187477 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233154091e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=937623552 dispatch_meta_bytes=458424 combine_payload_bytes=937623552 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938081976 ingress_bytes=1289945088 link_bytes=1289945088 token_bytes=1875705528 weight_movement_bytes=352321536 total_nvlink_bytes=2228027064 weight_to_token_ratio=0.187834 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223154043e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939982848 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879506944 weight_movement_bytes=880803840 total_nvlink_bytes=2760310784 weight_to_token_ratio=0.468636 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114672 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000122 router_tokens_per_expert=[4, 0, 0, 0, 0, 8192, 8186, 0, 0, 0, 8192, 8189, 8192, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16374, 16384, 16384, 16384] dup_rows_per_dst=[6, 0, 0, 10, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16378, 16384, 16374, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=458888 combine_payload_bytes=939147264 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939606152 ingress_bytes=1819951104 link_bytes=1819951104 token_bytes=1878753416 weight_movement_bytes=880803840 total_nvlink_bytes=2759557256 weight_to_token_ratio=0.468824 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019096650e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=937000960 dispatch_meta_bytes=458760 combine_payload_bytes=937000960 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937459720 ingress_bytes=1289322496 link_bytes=1289322496 token_bytes=1874460680 weight_movement_bytes=352321536 total_nvlink_bytes=2226782216 weight_to_token_ratio=0.187959 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.219667788e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114678 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8187] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] dup_rows_per_dst=[0, 0, 0, 10, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130234 payload_rows_remote=113850 dup_rows=838 dup_rows_remote=304 dedup_ratio=0.006393 router_tokens_per_expert=[9, 9, 14, 7, 55, 0, 0, 8184, 0, 9, 8186, 8182, 8, 61, 0, 0, 8185, 1, 0, 0, 7852, 0, 8139, 0, 8180, 0, 1, 278, 0, 8169, 6, 1] payload_rows_per_dst=[16384, 16376, 16372, 16294, 16370, 15704, 16384, 16350] dup_rows_per_dst=[66, 102, 10, 122, 2, 0, 534, 2] combine_rows_per_src=[16384, 16376, 16372, 16294, 16370, 15704, 16384, 16350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=458752 combine_payload_bytes=939393024 prefetch_bytes=0 reduce_bytes=0 egress_bytes=939851776 ingress_bytes=939393024 link_bytes=939851776 token_bytes=1879244800 weight_movement_bytes=0 total_nvlink_bytes=1879244800 weight_to_token_ratio=0.000000 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.262777632e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131040 payload_rows_remote=114656 dup_rows=32 dup_rows_remote=26 dedup_ratio=0.000244 router_tokens_per_expert=[13, 0, 0, 0, 0, 8192, 8186, 0, 0, 0, 8192, 8182, 8192, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16378, 16384, 16364, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[26, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[16378, 16384, 16364, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131002 payload_rows_remote=114624 dup_rows=70 dup_rows_remote=48 dedup_ratio=0.000534 router_tokens_per_expert=[4, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8189] payload_rows_per_dst=[16350, 16362, 16376, 16384, 16384, 16384, 16384, 16378] dup_rows_per_dst=[6, 22, 0, 20, 0, 0, 0, 22] combine_rows_per_src=[16350, 16362, 16376, 16384, 16384, 16384, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=932659200 dispatch_meta_bytes=456616 combine_payload_bytes=932659200 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=933115816 ingress_bytes=1813463040 link_bytes=1813463040 token_bytes=1865775016 weight_movement_bytes=880803840 total_nvlink_bytes=2746578856 weight_to_token_ratio=0.472085 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.015463605e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130680 payload_rows_remote=114308 dup_rows=392 dup_rows_remote=392 dedup_ratio=0.002991 router_tokens_per_expert=[2, 1, 3, 1, 7, 0, 1, 8190, 0, 2, 8191, 8184, 1, 76, 0, 6, 8186, 0, 0, 0, 8016, 0, 8186, 5, 8189, 1, 0, 98, 4, 8184, 1, 1] payload_rows_per_dst=[16384, 16382, 16372, 16374, 16372, 16034, 16384, 16378] dup_rows_per_dst=[12, 14, 0, 164, 0, 8, 192, 2] combine_rows_per_src=[16384, 16382, 16372, 16374, 16372, 16034, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458752 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939900928 ingress_bytes=1820246016 link_bytes=1820246016 token_bytes=1879343104 weight_movement_bytes=880803840 total_nvlink_bytes=2760146944 weight_to_token_ratio=0.468676 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019261788e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130130 payload_rows_remote=113746 dup_rows=942 dup_rows_remote=942 dedup_ratio=0.007187 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8192, 8192, 0, 58, 0, 8, 8186, 0, 0, 0, 8132, 0, 8192, 1, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16362, 16384, 16372, 16272, 15588, 16384] dup_rows_per_dst=[0, 24, 0, 132, 0, 786, 0, 0] combine_rows_per_src=[16384, 16384, 16362, 16384, 16372, 16272, 15588, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=458728 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=939720680 ingress_bytes=1996226560 link_bytes=1996226560 token_bytes=1878982632 weight_movement_bytes=1056964608 total_nvlink_bytes=2935947240 weight_to_token_ratio=0.562520 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117803547e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938999808 dispatch_meta_bytes=458688 combine_payload_bytes=938999808 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939458496 ingress_bytes=1291321344 link_bytes=1291321344 token_bytes=1878458304 weight_movement_bytes=352321536 total_nvlink_bytes=2230779840 weight_to_token_ratio=0.187559 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230860502e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130892 payload_rows_remote=114672 dup_rows=180 dup_rows_remote=180 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 72, 0, 18, 8184, 0, 0, 0, 8108, 0, 8192, 2, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16368, 16220, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 180, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16368, 16220, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=936411136 dispatch_meta_bytes=458800 combine_payload_bytes=936411136 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=936869936 ingress_bytes=1993375744 link_bytes=1993375744 token_bytes=1873281072 weight_movement_bytes=1056964608 total_nvlink_bytes=2930245680 weight_to_token_ratio=0.564232 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.116207209e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=931807232 dispatch_meta_bytes=458752 combine_payload_bytes=931807232 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=932265984 ingress_bytes=1284128768 link_bytes=1284128768 token_bytes=1864073216 weight_movement_bytes=352321536 total_nvlink_bytes=2216394752 weight_to_token_ratio=0.189006 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.190585079e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130612 payload_rows_remote=114234 dup_rows=460 dup_rows_remote=460 dedup_ratio=0.003510 router_tokens_per_expert=[0, 0, 0, 7, 7, 0, 78, 8, 227, 42, 7975, 8173, 4, 0, 0, 7, 8144, 8175, 7999, 9, 6, 8183, 0, 14, 0, 0, 5, 8184, 0, 10, 168, 8111] payload_rows_per_dst=[16360, 16384, 16356, 16364, 16016, 16384, 16378, 16370] dup_rows_per_dst=[0, 90, 132, 8, 0, 22, 0, 208] combine_rows_per_src=[16360, 16384, 16356, 16364, 16016, 16384, 16378, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=459408 combine_payload_bytes=939393024 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939852432 ingress_bytes=1644036096 link_bytes=1644036096 token_bytes=1879245456 weight_movement_bytes=704643072 total_nvlink_bytes=2583888528 weight_to_token_ratio=0.374961 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205915883e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130970 payload_rows_remote=114586 dup_rows=102 dup_rows_remote=102 dedup_ratio=0.000778 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8191, 0, 48, 0, 2, 8192, 1, 0, 0, 8142, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16284, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 100, 2, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16284, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129622 payload_rows_remote=113238 dup_rows=1450 dup_rows_remote=2 dedup_ratio=0.011063 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 694, 0, 30, 8189, 0, 0, 0, 7467, 0, 8192, 4, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 14940, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 1448, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 14940, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128080 payload_rows_remote=111696 dup_rows=2992 dup_rows_remote=2956 dedup_ratio=0.022827 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8191, 0, 38, 0, 4, 8192, 0, 0, 0, 8151, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16306, 16254, 16382, 13672, 16384, 16318, 16380, 16384] dup_rows_per_dst=[0, 0, 40, 68, 62, 2712, 74, 36] combine_rows_per_src=[16306, 16254, 16382, 13672, 16384, 16318, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=935804928 dispatch_meta_bytes=458776 combine_payload_bytes=935804928 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=936263704 ingress_bytes=1111965696 link_bytes=1111965696 token_bytes=1872068632 weight_movement_bytes=176160768 total_nvlink_bytes=2048229400 weight_to_token_ratio=0.094100 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.226543740e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130962 payload_rows_remote=114580 dup_rows=110 dup_rows_remote=94 dedup_ratio=0.000839 router_tokens_per_expert=[1, 0, 2, 1, 1, 0, 13, 1, 131, 4, 8064, 8186, 1, 0, 0, 1, 8186, 8190, 8141, 1, 1, 8189, 0, 2, 0, 0, 1, 8191, 1, 2, 46, 8179] payload_rows_per_dst=[16380, 16384, 16382, 16382, 16284, 16384, 16384, 16382] dup_rows_per_dst=[0, 18, 16, 2, 0, 0, 0, 74] combine_rows_per_src=[16380, 16384, 16382, 16382, 16284, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938688512 dispatch_meta_bytes=458752 combine_payload_bytes=938688512 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939147264 ingress_bytes=1819492352 link_bytes=1819492352 token_bytes=1877835776 weight_movement_bytes=880803840 total_nvlink_bytes=2758639616 weight_to_token_ratio=0.469053 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018839768e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130672 payload_rows_remote=114288 dup_rows=400 dup_rows_remote=378 dedup_ratio=0.003052 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16384, 16368, 16384, 16008] dup_rows_per_dst=[22, 0, 0, 12, 366, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16376, 16384, 16384, 16368, 16384, 16008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[2, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 8192, 8191, 8192, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=927645696 dispatch_meta_bytes=452960 combine_payload_bytes=927645696 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928098656 ingress_bytes=1279967232 link_bytes=1279967232 token_bytes=1855744352 weight_movement_bytes=352321536 total_nvlink_bytes=2208065888 weight_to_token_ratio=0.189855 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.167282215e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=915013632 dispatch_meta_bytes=458608 combine_payload_bytes=915013632 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=915472240 ingress_bytes=1267335168 link_bytes=1267335168 token_bytes=1830485872 weight_movement_bytes=352321536 total_nvlink_bytes=2182807408 weight_to_token_ratio=0.192474 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.096547929e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114682 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000046 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 8189, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[6, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938639360 dispatch_meta_bytes=458696 combine_payload_bytes=938639360 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=939098056 ingress_bytes=2171764736 link_bytes=2171764736 token_bytes=1877737416 weight_movement_bytes=1233125376 total_nvlink_bytes=3110862792 weight_to_token_ratio=0.656708 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216097598e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=936247296 dispatch_meta_bytes=458664 combine_payload_bytes=936247296 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=936705960 ingress_bytes=1640890368 link_bytes=1640890368 token_bytes=1872953256 weight_movement_bytes=704643072 total_nvlink_bytes=2577596328 weight_to_token_ratio=0.376220 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.188301119e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879441408 weight_movement_bytes=352321536 total_nvlink_bytes=2231762944 weight_to_token_ratio=0.187461 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=458752 combine_payload_bytes=939474944 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=939933696 ingress_bytes=1291796480 link_bytes=1291796480 token_bytes=1879408640 weight_movement_bytes=352321536 total_nvlink_bytes=2231730176 weight_to_token_ratio=0.187464 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233521066e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130808 payload_rows_remote=114580 dup_rows=264 dup_rows_remote=264 dedup_ratio=0.002014 router_tokens_per_expert=[6, 5, 7, 21, 15, 7, 0, 8177, 6, 8182, 8, 8, 8183, 10, 0, 4, 8181, 0, 59, 8138, 9, 0, 0, 8180, 8107, 0, 0, 7, 7, 26, 0, 8183] payload_rows_per_dst=[16384, 16368, 16380, 16380, 16310, 16378, 16228, 16380] dup_rows_per_dst=[56, 30, 28, 14, 84, 0, 0, 52] combine_rows_per_src=[16384, 16368, 16380, 16380, 16310, 16378, 16228, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130848 payload_rows_remote=114464 dup_rows=224 dup_rows_remote=114 dedup_ratio=0.001709 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16276, 16274, 16384, 16384, 16378, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 108, 0, 6, 110] combine_rows_per_src=[16384, 16276, 16274, 16384, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938639360 dispatch_meta_bytes=459376 combine_payload_bytes=938639360 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939098736 ingress_bytes=1290960896 link_bytes=1290960896 token_bytes=1877738096 weight_movement_bytes=352321536 total_nvlink_bytes=2230059632 weight_to_token_ratio=0.187631 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228842144e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130866 payload_rows_remote=114490 dup_rows=206 dup_rows_remote=202 dedup_ratio=0.001572 router_tokens_per_expert=[1, 0, 1, 1, 2, 1, 0, 8190, 1, 8187, 1, 1, 8191, 1, 0, 1, 8187, 0, 19, 8116, 1, 0, 0, 8191, 8172, 4, 0, 0, 1, 75, 0, 8191] payload_rows_per_dst=[16380, 16382, 16376, 16384, 16232, 16384, 16344, 16384] dup_rows_per_dst=[0, 4, 4, 2, 38, 0, 8, 150] combine_rows_per_src=[16380, 16382, 16376, 16384, 16232, 16384, 16344, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128394 payload_rows_remote=112010 dup_rows=2678 dup_rows_remote=2674 dedup_ratio=0.020432 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 8189, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 7988, 0, 0, 0, 8192, 8192, 1, 0, 0, 0, 206, 0, 8192] payload_rows_per_dst=[16384, 16384, 16378, 16376, 16356, 16384, 16384, 13748] dup_rows_per_dst=[4, 10, 0, 0, 2454, 0, 184, 26] combine_rows_per_src=[16384, 16384, 16378, 16376, 16356, 16384, 16384, 13748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879506944 weight_movement_bytes=352321536 total_nvlink_bytes=2231828480 weight_to_token_ratio=0.187454 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879506944 weight_movement_bytes=528482304 total_nvlink_bytes=2407989248 weight_to_token_ratio=0.281181 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130472 payload_rows_remote=114094 dup_rows=600 dup_rows_remote=600 dedup_ratio=0.004578 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 292, 0, 7, 8189, 0, 0, 0, 7890, 0, 8192, 6, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 15790, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 598, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 15790, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130470 payload_rows_remote=114086 dup_rows=602 dup_rows_remote=602 dedup_ratio=0.004593 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 8192, 0, 8188, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 7895, 0, 0, 0, 8192, 8192, 2, 0, 0, 0, 297, 0, 8192] payload_rows_per_dst=[16384, 16384, 16376, 16384, 15790, 16384, 16384, 16384] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 4, 594] combine_rows_per_src=[16384, 16384, 16376, 16384, 15790, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=937689088 dispatch_meta_bytes=458312 combine_payload_bytes=937689088 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=938147400 ingress_bytes=1642332160 link_bytes=1642332160 token_bytes=1875836488 weight_movement_bytes=704643072 total_nvlink_bytes=2580479560 weight_to_token_ratio=0.375642 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196374553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937902080 dispatch_meta_bytes=458768 combine_payload_bytes=937902080 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938360848 ingress_bytes=1290223616 link_bytes=1290223616 token_bytes=1876262928 weight_movement_bytes=352321536 total_nvlink_bytes=2228584464 weight_to_token_ratio=0.187778 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224713684e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=917585920 dispatch_meta_bytes=458736 combine_payload_bytes=917585920 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=918044656 ingress_bytes=1446068224 link_bytes=1446068224 token_bytes=1835630576 weight_movement_bytes=528482304 total_nvlink_bytes=2364112880 weight_to_token_ratio=0.287902 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.097378436e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130180 payload_rows_remote=113798 dup_rows=892 dup_rows_remote=892 dedup_ratio=0.006805 router_tokens_per_expert=[91, 8186, 7893, 8150, 9, 13, 14, 386, 5, 0, 8162, 8181, 8, 8181, 0, 10, 0, 6, 11, 0, 0, 10, 8028, 0, 0, 0, 8184, 7, 0, 1, 0, 0] payload_rows_per_dst=[15936, 16384, 16334, 16384, 16382, 16076, 16382, 16302] dup_rows_per_dst=[32, 832, 0, 14, 14, 0, 0, 0] combine_rows_per_src=[15936, 16384, 16334, 16384, 16382, 16076, 16382, 16302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934658048 dispatch_meta_bytes=458776 combine_payload_bytes=934658048 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935116824 ingress_bytes=1286979584 link_bytes=1286979584 token_bytes=1869774872 weight_movement_bytes=352321536 total_nvlink_bytes=2222096408 weight_to_token_ratio=0.188430 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.206548459e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=934592512 dispatch_meta_bytes=458752 combine_payload_bytes=934592512 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=935051264 ingress_bytes=1110753280 link_bytes=1110753280 token_bytes=1869643776 weight_movement_bytes=176160768 total_nvlink_bytes=2045804544 weight_to_token_ratio=0.094222 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.219754717e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=932233216 dispatch_meta_bytes=458760 combine_payload_bytes=932233216 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=932691976 ingress_bytes=1108393984 link_bytes=1108393984 token_bytes=1864925192 weight_movement_bytes=176160768 total_nvlink_bytes=2041085960 weight_to_token_ratio=0.094460 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.206543644e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130720 payload_rows_remote=114336 dup_rows=352 dup_rows_remote=352 dedup_ratio=0.002686 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8016, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 176, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16032, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 352] combine_rows_per_src=[16384, 16384, 16384, 16384, 16032, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130648 payload_rows_remote=114264 dup_rows=424 dup_rows_remote=424 dedup_ratio=0.003235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 8187, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 7985, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 212, 0, 8192] payload_rows_per_dst=[16384, 16384, 16374, 16384, 15970, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 424] combine_rows_per_src=[16384, 16384, 16374, 16384, 15970, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130866 payload_rows_remote=114490 dup_rows=206 dup_rows_remote=206 dedup_ratio=0.001572 router_tokens_per_expert=[19, 8180, 8128, 8181, 2, 2, 4, 88, 1, 0, 8187, 8190, 1, 8190, 0, 0, 0, 1, 1, 0, 4, 1, 8164, 0, 0, 0, 8191, 1, 0, 0, 0, 0] payload_rows_per_dst=[16274, 16376, 16376, 16382, 16382, 16330, 16384, 16362] dup_rows_per_dst=[20, 176, 0, 0, 2, 8, 0, 0] combine_rows_per_src=[16274, 16376, 16376, 16382, 16382, 16330, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130304 payload_rows_remote=113920 dup_rows=768 dup_rows_remote=68 dedup_ratio=0.005859 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 7854, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 338, 0, 8192] payload_rows_per_dst=[16328, 16354, 16382, 16380, 15708, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 6, 0, 56, 6, 0, 700] combine_rows_per_src=[16328, 16354, 16382, 16380, 15708, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28257647696 prefetch_bytes=5284823040 reduce_bytes=4404019200 weight_movement_bytes=9688842240 egress_bytes=14132444240 ingress_bytes=23814045696 link_bytes=23814045696 total_nvlink_bytes=37946489936 weight_to_token_ratio=0.342875 prefetch_share_of_ingress=0.221920 dedup_ratio=0.053237 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=25 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.021070 gemm_inner_pad_upper_bound=18415 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.334294714e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129972 payload_rows_remote=113588 dup_rows=1100 dup_rows_remote=12 dedup_ratio=0.008392 router_tokens_per_expert=[44, 8157, 8131, 8192, 0, 0, 0, 51, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15306, 16384, 16372, 16374, 16384, 16384, 16384] dup_rows_per_dst=[1088, 0, 10, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 15306, 16384, 16372, 16374, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=936640512 dispatch_meta_bytes=458752 combine_payload_bytes=936640512 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=937099264 ingress_bytes=1641283584 link_bytes=1641283584 token_bytes=1873739776 weight_movement_bytes=704643072 total_nvlink_bytes=2578382848 weight_to_token_ratio=0.376062 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.190502965e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=936050688 dispatch_meta_bytes=458752 combine_payload_bytes=936050688 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=936509440 ingress_bytes=1816854528 link_bytes=1816854528 token_bytes=1872560128 weight_movement_bytes=880803840 total_nvlink_bytes=2753363968 weight_to_token_ratio=0.470374 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.017362696e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131044 payload_rows_remote=114660 dup_rows=28 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[6, 8144, 8174, 8192, 0, 0, 0, 53, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16360, 16380, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 14, 0, 0, 0, 14, 0, 0] combine_rows_per_src=[16360, 16380, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937902080 dispatch_meta_bytes=458784 combine_payload_bytes=937902080 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=938360864 ingress_bytes=1818705920 link_bytes=1818705920 token_bytes=1876262944 weight_movement_bytes=880803840 total_nvlink_bytes=2757066784 weight_to_token_ratio=0.469446 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018399399e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=933232640 dispatch_meta_bytes=455952 combine_payload_bytes=933232640 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=933688592 ingress_bytes=1990197248 link_bytes=1990197248 token_bytes=1866921232 weight_movement_bytes=1056964608 total_nvlink_bytes=2923885840 weight_to_token_ratio=0.566154 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.114427384e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=11 export_slots=11 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 1, 0, 2, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=930512896 dispatch_meta_bytes=454400 combine_payload_bytes=930512896 prefetch_bytes=176160768 reduce_bytes=1937768448 egress_bytes=930967296 ingress_bytes=3044442112 link_bytes=3044442112 token_bytes=1861480192 weight_movement_bytes=2113929216 total_nvlink_bytes=3975409408 weight_to_token_ratio=1.135617 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.024621 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.704760501e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=458696 combine_payload_bytes=939294720 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939753416 ingress_bytes=1291616256 link_bytes=1291616256 token_bytes=1879048136 weight_movement_bytes=352321536 total_nvlink_bytes=2231369672 weight_to_token_ratio=0.187500 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232511886e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 3, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29057969608 prefetch_bytes=4580179968 reduce_bytes=7927234560 weight_movement_bytes=12507414528 egress_bytes=14532668872 ingress_bytes=27032715264 link_bytes=27032715264 total_nvlink_bytes=41565384136 weight_to_token_ratio=0.430430 prefetch_share_of_ingress=0.169431 dedup_ratio=0.034537 prefetch_slots=52 unused_prefetch_slots=204 slot_utilization=0.203125 reduce_slots=45 remote_demand=26 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019413 gemm_inner_pad_upper_bound=20701 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.460745092e-05 prefetch_bytes_per_gemm_flop=1.602943497e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28403637552 prefetch_bytes=5284823040 reduce_bytes=6165626880 weight_movement_bytes=11450449920 egress_bytes=14205410608 ingress_bytes=25648676864 link_bytes=25648676864 total_nvlink_bytes=39854087472 weight_to_token_ratio=0.403133 prefetch_share_of_ingress=0.206047 dedup_ratio=0.048705 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=35 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020005 gemm_inner_pad_upper_bound=20447 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.976367760e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114680 dup_rows=8 dup_rows_remote=0 dedup_ratio=0.000061 router_tokens_per_expert=[234, 8173, 7955, 8192, 0, 0, 0, 23, 0, 0, 8192, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 8, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=939982848 ingress_bytes=2172649472 link_bytes=2172649472 token_bytes=1879506944 weight_movement_bytes=1233125376 total_nvlink_bytes=3112632320 weight_to_token_ratio=0.656090 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216593013e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131052 payload_rows_remote=114668 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000153 router_tokens_per_expert=[14, 8160, 8168, 8192, 0, 0, 0, 38, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16364, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 12, 0, 0, 0, 8, 0, 0] combine_rows_per_src=[16364, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28881188552 prefetch_bytes=4404019200 reduce_bytes=3523215360 weight_movement_bytes=7927234560 egress_bytes=14444246728 ingress_bytes=22364176384 link_bytes=22364635136 total_nvlink_bytes=36808423112 weight_to_token_ratio=0.274477 prefetch_share_of_ingress=0.196923 dedup_ratio=0.039658 prefetch_slots=50 unused_prefetch_slots=206 slot_utilization=0.195312 reduce_slots=20 remote_demand=25 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.021899 gemm_inner_pad_upper_bound=18415 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.827038831e-05 prefetch_bytes_per_gemm_flop=1.541291824e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130548 payload_rows_remote=114164 dup_rows=524 dup_rows_remote=420 dedup_ratio=0.003998 router_tokens_per_expert=[52, 8161, 8122, 8192, 0, 0, 0, 50, 0, 0, 8192, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[15866, 16384, 16382, 16382, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 38, 50, 2, 2, 326, 2, 104] combine_rows_per_src=[15866, 16384, 16382, 16382, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458720 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939917280 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879375840 weight_movement_bytes=352321536 total_nvlink_bytes=2231697376 weight_to_token_ratio=0.187467 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458752 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939819008 ingress_bytes=1467842560 link_bytes=1467842560 token_bytes=1879179264 weight_movement_bytes=528482304 total_nvlink_bytes=2407661568 weight_to_token_ratio=0.281230 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219305628e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=935231488 dispatch_meta_bytes=458336 combine_payload_bytes=935231488 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=935689824 ingress_bytes=1639874560 link_bytes=1639874560 token_bytes=1870921312 weight_movement_bytes=704643072 total_nvlink_bytes=2575564384 weight_to_token_ratio=0.376629 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.182613019e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28912426728 prefetch_bytes=5284823040 reduce_bytes=4051697664 weight_movement_bytes=9336520704 egress_bytes=14459838184 ingress_bytes=23789109248 link_bytes=23789109248 total_nvlink_bytes=38248947432 weight_to_token_ratio=0.322924 prefetch_share_of_ingress=0.222153 dedup_ratio=0.034705 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=23 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019768 gemm_inner_pad_upper_bound=19685 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.325567608e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28957987120 prefetch_bytes=4932501504 reduce_bytes=4580179968 weight_movement_bytes=9512681472 egress_bytes=14482657584 ingress_bytes=23988011008 link_bytes=23988011008 total_nvlink_bytes=38470668592 weight_to_token_ratio=0.328499 prefetch_share_of_ingress=0.205624 dedup_ratio=0.035676 prefetch_slots=56 unused_prefetch_slots=200 slot_utilization=0.218750 reduce_slots=26 remote_demand=28 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020123 gemm_inner_pad_upper_bound=20066 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.395178035e-05 prefetch_bytes_per_gemm_flop=1.726246843e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130730 payload_rows_remote=114688 dup_rows=342 dup_rows_remote=342 dedup_ratio=0.002609 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8021, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 171, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16042, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 342] combine_rows_per_src=[16384, 16384, 16384, 16384, 16042, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27666261280 prefetch_bytes=5460983808 reduce_bytes=6517948416 weight_movement_bytes=11978932224 egress_bytes=13836707104 ingress_bytes=25808486400 link_bytes=25808486400 total_nvlink_bytes=39645193504 weight_to_token_ratio=0.432980 prefetch_share_of_ingress=0.211596 dedup_ratio=0.070571 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=37 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.021070 gemm_inner_pad_upper_bound=18796 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.032296928e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[3, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=460120 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939984216 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879508312 weight_movement_bytes=880803840 total_nvlink_bytes=2760312152 weight_to_token_ratio=0.468635 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114674 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000107 router_tokens_per_expert=[91, 8178, 8094, 8192, 0, 0, 0, 19, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 10, 0, 0, 0, 4, 0, 0] combine_rows_per_src=[16370, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=458752 combine_payload_bytes=939409408 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939868160 ingress_bytes=1467891712 link_bytes=1467891712 token_bytes=1879277568 weight_movement_bytes=528482304 total_nvlink_bytes=2407759872 weight_to_token_ratio=0.281216 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219580859e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28829723304 prefetch_bytes=5637144576 reduce_bytes=3699376128 weight_movement_bytes=9336520704 egress_bytes=14418520744 ingress_bytes=23747723264 link_bytes=23747723264 total_nvlink_bytes=38166244008 weight_to_token_ratio=0.323851 prefetch_share_of_ingress=0.237376 dedup_ratio=0.036987 prefetch_slots=64 unused_prefetch_slots=192 slot_utilization=0.250000 reduce_slots=21 remote_demand=32 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019768 gemm_inner_pad_upper_bound=19812 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.311083593e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 anomalies=none - [2026-09-16 21:37:08] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 569.6 | throughput per GPU (TFLOP/s/GPU): 585.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.278158E+00 | loss scale: 1.0 | grad norm: 5.711 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96440 payload_rows_remote=80266 dup_rows=34632 dup_rows_remote=25460 dedup_ratio=0.264221 router_tokens_per_expert=[4376, 0, 1915, 1692, 1974, 0, 2846, 1003, 24, 1537, 1, 2499, 2553, 2888, 646, 169, 0, 7, 2069, 0, 2, 1439, 4515, 611, 2, 2330, 7931, 2410, 4078, 5502, 3594, 6923] payload_rows_per_dst=[11990, 9026, 9096, 9896, 14786, 11298, 16174, 14174] dup_rows_per_dst=[3976, 2620, 1944, 2616, 3212, 1836, 9172, 9256] combine_rows_per_src=[11990, 9026, 9096, 9896, 14786, 11298, 16174, 14174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96364 payload_rows_remote=83458 dup_rows=34708 dup_rows_remote=27554 dedup_ratio=0.264801 router_tokens_per_expert=[4933, 0, 1308, 1963, 2440, 0, 3096, 1075, 30, 2650, 5, 2289, 1944, 3351, 889, 221, 3, 0, 1320, 0, 2, 2032, 4234, 447, 0, 1892, 8025, 2247, 3791, 5643, 3211, 6495] payload_rows_per_dst=[12360, 12906, 14582, 14698, 13806, 11428, 8002, 8582] dup_rows_per_dst=[4048, 7154, 6652, 10020, 1830, 2002, 1278, 1724] combine_rows_per_src=[12360, 12906, 14582, 14698, 13806, 11428, 8002, 8582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94920 payload_rows_remote=81878 dup_rows=36152 dup_rows_remote=30576 dedup_ratio=0.275818 router_tokens_per_expert=[4883, 3, 1856, 1822, 3709, 2, 2189, 1084, 92, 1656, 8, 3031, 2675, 2588, 684, 270, 22, 8, 2266, 6, 1, 1697, 4046, 1015, 2, 2021, 7895, 1952, 3648, 4342, 3007, 7056] payload_rows_per_dst=[13042, 14972, 12952, 13444, 15014, 11164, 8370, 5962] dup_rows_per_dst=[5576, 10502, 5306, 6600, 3702, 2354, 2112, 0] combine_rows_per_src=[13042, 14972, 12952, 13444, 15014, 11164, 8370, 5962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95188 payload_rows_remote=85112 dup_rows=35884 dup_rows_remote=32196 dedup_ratio=0.273773 router_tokens_per_expert=[4576, 0, 1935, 1780, 3119, 2, 2928, 1440, 12, 1979, 0, 2086, 2412, 2538, 642, 215, 3, 2, 1726, 1, 0, 1715, 5114, 627, 1, 1872, 7954, 1866, 4119, 5183, 3180, 6509] payload_rows_per_dst=[12464, 11068, 13782, 10076, 14090, 15832, 6614, 11262] dup_rows_per_dst=[4118, 3910, 4738, 3688, 2392, 12838, 864, 3336] combine_rows_per_src=[12464, 11068, 13782, 10076, 14090, 15832, 6614, 11262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97782 payload_rows_remote=86658 dup_rows=33290 dup_rows_remote=31480 dedup_ratio=0.253983 router_tokens_per_expert=[4225, 1, 1397, 1715, 2898, 0, 2761, 794, 8, 2388, 2, 2202, 1723, 3741, 558, 279, 4, 15, 1911, 0, 0, 1761, 4205, 501, 0, 1938, 7586, 2030, 4557, 5994, 3442, 6900] payload_rows_per_dst=[11300, 9696, 11756, 10030, 14684, 11124, 15886, 13306] dup_rows_per_dst=[3376, 3210, 4514, 2572, 2976, 1810, 7222, 7610] combine_rows_per_src=[11300, 9696, 11756, 10030, 14684, 11124, 15886, 13306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[1, 1, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=91836 payload_rows_remote=76354 dup_rows=39236 dup_rows_remote=23860 dedup_ratio=0.299347 router_tokens_per_expert=[4366, 1, 1790, 2068, 3198, 1, 2318, 1257, 13, 2146, 2, 2532, 2158, 2748, 371, 258, 13, 1, 1668, 0, 2, 1644, 4245, 811, 0, 2141, 7928, 2365, 4398, 4974, 3069, 7050] payload_rows_per_dst=[11982, 10058, 8264, 8994, 9596, 11374, 16086, 15482] dup_rows_per_dst=[4468, 3490, 1122, 2076, 1892, 2030, 8782, 15376] combine_rows_per_src=[11982, 10058, 8264, 8994, 9596, 11374, 16086, 15482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=657539072 dispatch_meta_bytes=422904 combine_payload_bytes=657539072 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=657961976 ingress_bytes=2066825216 link_bytes=2066825216 token_bytes=1315501048 weight_movement_bytes=1409286144 total_nvlink_bytes=2724787192 weight_to_token_ratio=1.071292 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=15232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.157335847e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=683687936 dispatch_meta_bytes=444048 combine_payload_bytes=683687936 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=684131984 ingress_bytes=1036009472 link_bytes=1036009472 token_bytes=1367819920 weight_movement_bytes=352321536 total_nvlink_bytes=1720141456 weight_to_token_ratio=0.257579 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=12544 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.801220592e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=670744576 dispatch_meta_bytes=449816 combine_payload_bytes=670744576 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=671194392 ingress_bytes=1023066112 link_bytes=1023066112 token_bytes=1341938968 weight_movement_bytes=352321536 total_nvlink_bytes=1694260504 weight_to_token_ratio=0.262547 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=19840 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.728743178e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97776 payload_rows_remote=84464 dup_rows=33296 dup_rows_remote=28932 dedup_ratio=0.254028 router_tokens_per_expert=[5490, 0, 1708, 1853, 3220, 1, 2758, 1176, 56, 1750, 0, 1875, 2203, 2261, 715, 168, 0, 14, 1676, 1, 0, 1255, 4330, 688, 2, 2630, 8008, 2595, 4374, 5157, 2896, 6676] payload_rows_per_dst=[13362, 11576, 13312, 15370, 14364, 11338, 8992, 9462] dup_rows_per_dst=[4740, 5368, 4364, 9316, 2370, 3232, 1462, 2444] combine_rows_per_src=[13362, 11576, 13312, 15370, 14364, 11338, 8992, 9462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=697237504 dispatch_meta_bytes=469232 combine_payload_bytes=697237504 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=697706736 ingress_bytes=1049559040 link_bytes=1049559040 token_bytes=1394944240 weight_movement_bytes=352321536 total_nvlink_bytes=1747265776 weight_to_token_ratio=0.252570 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=12672 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.877092516e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=709902336 dispatch_meta_bytes=472552 combine_payload_bytes=709902336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=710374888 ingress_bytes=1062223872 link_bytes=1062223872 token_bytes=1420277224 weight_movement_bytes=352321536 total_nvlink_bytes=1772598760 weight_to_token_ratio=0.248065 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=18176 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.948010289e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=625491968 dispatch_meta_bytes=400856 combine_payload_bytes=625491968 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=625892824 ingress_bytes=2034778112 link_bytes=2034778112 token_bytes=1251384792 weight_movement_bytes=1409286144 total_nvlink_bytes=2660670936 weight_to_token_ratio=1.126181 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.139390807e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=691929088 dispatch_meta_bytes=453584 combine_payload_bytes=691929088 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=692382672 ingress_bytes=1044250624 link_bytes=1044250624 token_bytes=1384311760 weight_movement_bytes=352321536 total_nvlink_bytes=1736633296 weight_to_token_ratio=0.254510 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=16896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.847367603e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96460 payload_rows_remote=82348 dup_rows=34612 dup_rows_remote=32366 dedup_ratio=0.264069 router_tokens_per_expert=[3917, 0, 1374, 1844, 3685, 5, 2887, 1181, 19, 1972, 0, 1839, 1737, 3306, 573, 222, 4, 1, 1495, 0, 0, 2258, 4517, 677, 0, 3153, 7956, 2173, 3240, 5514, 3308, 6679] payload_rows_per_dst=[10894, 11178, 13706, 9534, 14112, 13428, 13152, 10456] dup_rows_per_dst=[3376, 4338, 4982, 2142, 2246, 7984, 6904, 2640] combine_rows_per_src=[10894, 11178, 13706, 9534, 14112, 13428, 13152, 10456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121362 payload_rows_remote=104980 dup_rows=9710 dup_rows_remote=9706 dedup_ratio=0.074081 router_tokens_per_expert=[4325, 0, 8, 0, 8192, 0, 0, 0, 8191, 1525, 8182, 479, 4122, 2, 5930, 8189, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 8192, 0, 2] payload_rows_per_dst=[8662, 16384, 16372, 14412, 16384, 16382, 16382, 16384] dup_rows_per_dst=[4, 0, 4000, 5696, 0, 0, 4, 6] combine_rows_per_src=[8662, 16384, 16372, 14412, 16384, 16382, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119168 payload_rows_remote=102802 dup_rows=11904 dup_rows_remote=11904 dedup_ratio=0.090820 router_tokens_per_expert=[4646, 0, 8, 8, 8183, 0, 0, 0, 8189, 1218, 8176, 464, 3119, 9, 6957, 8174, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 8191, 0, 1] payload_rows_per_dst=[16384, 16366, 12690, 8244, 16374, 16378, 16348, 16384] dup_rows_per_dst=[9292, 0, 1580, 1020, 0, 0, 12, 0] combine_rows_per_src=[16384, 16366, 12690, 8244, 16374, 16378, 16348, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121050 payload_rows_remote=104712 dup_rows=10022 dup_rows_remote=4186 dedup_ratio=0.076462 router_tokens_per_expert=[2937, 0, 15, 3, 8174, 0, 5, 0, 8183, 3036, 8132, 2048, 3734, 13, 4683, 8168, 8187, 5, 4, 0, 4, 0, 5, 0, 5, 1, 5, 2, 0, 8185, 2, 0] payload_rows_per_dst=[16338, 16360, 15384, 7494, 16382, 16368, 16352, 16372] dup_rows_per_dst=[5836, 46, 4110, 2, 10, 8, 10, 0] combine_rows_per_src=[16338, 16360, 15384, 7494, 16382, 16368, 16352, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118216 payload_rows_remote=104134 dup_rows=12856 dup_rows_remote=7714 dedup_ratio=0.098083 router_tokens_per_expert=[3683, 3, 4, 0, 8184, 3, 4, 0, 8187, 1475, 8157, 1629, 3536, 6, 6070, 8187, 8188, 1, 3, 3, 4, 2, 2, 0, 0, 15, 1, 0, 0, 8187, 1, 1] payload_rows_per_dst=[15730, 16376, 6502, 14082, 16384, 16382, 16382, 16378] dup_rows_per_dst=[7038, 6, 632, 5142, 6, 8, 24, 0] combine_rows_per_src=[15730, 16376, 6502, 14082, 16384, 16382, 16382, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 4, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120534 payload_rows_remote=104150 dup_rows=10538 dup_rows_remote=10538 dedup_ratio=0.080399 router_tokens_per_expert=[4756, 0, 0, 2, 8192, 0, 0, 0, 8192, 3034, 8192, 577, 2843, 0, 5198, 8163, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8191, 0, 0] payload_rows_per_dst=[9512, 16384, 16384, 12774, 16384, 16384, 16330, 16382] dup_rows_per_dst=[4, 0, 7222, 3308, 0, 0, 4, 0] combine_rows_per_src=[9512, 16384, 16384, 12774, 16384, 16384, 16330, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=859996160 dispatch_meta_bytes=458744 combine_payload_bytes=859996160 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=860454904 ingress_bytes=1564639232 link_bytes=1564639232 token_bytes=1720451064 weight_movement_bytes=704643072 total_nvlink_bytes=2425094136 weight_to_token_ratio=0.409569 gemm_rows_defined=66816 gemm_rows_charged=67584 gemm_tail_pad_rows=768 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.011364 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.761326587e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=842153984 dispatch_meta_bytes=458824 combine_payload_bytes=842153984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=842612808 ingress_bytes=1194475520 link_bytes=1194475520 token_bytes=1684766792 weight_movement_bytes=352321536 total_nvlink_bytes=2037088328 weight_to_token_ratio=0.209122 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.688564313e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=857800704 dispatch_meta_bytes=435592 combine_payload_bytes=857800704 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=858236296 ingress_bytes=1210122240 link_bytes=1210122240 token_bytes=1716037000 weight_movement_bytes=352321536 total_nvlink_bytes=2068358536 weight_to_token_ratio=0.205311 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=18688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.776179414e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=853065728 dispatch_meta_bytes=447392 combine_payload_bytes=853065728 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=853513120 ingress_bytes=2614673408 link_bytes=2614673408 token_bytes=1706578848 weight_movement_bytes=1761607680 total_nvlink_bytes=3468186528 weight_to_token_ratio=1.032245 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.464107967e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119734 payload_rows_remote=103354 dup_rows=11338 dup_rows_remote=11312 dedup_ratio=0.086502 router_tokens_per_expert=[3436, 0, 5, 2, 8192, 0, 0, 0, 8192, 813, 8180, 1902, 4360, 1, 5897, 8171, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8190, 0, 0] payload_rows_per_dst=[6880, 16384, 16370, 14604, 16382, 16384, 16350, 16380] dup_rows_per_dst=[6, 0, 5404, 5762, 0, 16, 124, 26] combine_rows_per_src=[6880, 16384, 16370, 14604, 16382, 16384, 16350, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=674594816 dispatch_meta_bytes=458856 combine_payload_bytes=674594816 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=675053672 ingress_bytes=1026916352 link_bytes=1026916352 token_bytes=1349648488 weight_movement_bytes=352321536 total_nvlink_bytes=1701970024 weight_to_token_ratio=0.261047 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=26112 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.750302915e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118064 payload_rows_remote=112936 dup_rows=13008 dup_rows_remote=12896 dedup_ratio=0.099243 router_tokens_per_expert=[3594, 0, 3, 0, 8192, 0, 0, 0, 8191, 2257, 8127, 363, 3945, 5, 6285, 8180, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 1, 0, 8191, 0, 1] payload_rows_per_dst=[16354, 16384, 5128, 14678, 16382, 16382, 16374, 16382] dup_rows_per_dst=[7094, 0, 112, 5792, 0, 0, 8, 2] combine_rows_per_src=[16354, 16384, 5128, 14678, 16382, 16382, 16374, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=853196800 dispatch_meta_bytes=458752 combine_payload_bytes=853196800 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=853655552 ingress_bytes=1381679104 link_bytes=1381679104 token_bytes=1706852352 weight_movement_bytes=528482304 total_nvlink_bytes=2235334656 weight_to_token_ratio=0.309624 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.736826241e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 3, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=846675968 dispatch_meta_bytes=458664 combine_payload_bytes=846675968 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=847134632 ingress_bytes=1198997504 link_bytes=1198997504 token_bytes=1693810600 weight_movement_bytes=352321536 total_nvlink_bytes=2046132136 weight_to_token_ratio=0.208005 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.713885536e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=925171712 dispatch_meta_bytes=503328 combine_payload_bytes=925171712 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=925675040 ingress_bytes=2158297088 link_bytes=2158297088 token_bytes=1850846752 weight_movement_bytes=1233125376 total_nvlink_bytes=3083972128 weight_to_token_ratio=0.666249 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=16896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.208556278e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123734 payload_rows_remote=107366 dup_rows=7338 dup_rows_remote=7338 dedup_ratio=0.055984 router_tokens_per_expert=[1, 4, 8192, 0, 0, 0, 8156, 0, 4568, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 8191, 0, 0, 0, 8184, 0, 3661, 0, 2] payload_rows_per_dst=[16384, 16312, 9136, 16384, 16384, 16382, 16368, 16384] dup_rows_per_dst=[10, 0, 0, 0, 2, 0, 0, 7326] combine_rows_per_src=[16384, 16312, 9136, 16384, 16384, 16382, 16368, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122508 payload_rows_remote=106136 dup_rows=8564 dup_rows_remote=8502 dedup_ratio=0.065338 router_tokens_per_expert=[4, 28, 8192, 0, 36, 1, 8180, 0, 4219, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 8153, 0, 3955, 0, 2] payload_rows_per_dst=[16384, 16372, 16384, 16384, 16384, 16380, 16306, 7914] dup_rows_per_dst=[64, 62, 8438, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16372, 16384, 16384, 16384, 16380, 16306, 7914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121674 payload_rows_remote=105290 dup_rows=9398 dup_rows_remote=9324 dedup_ratio=0.071701 router_tokens_per_expert=[9, 16, 8183, 5, 9, 0, 8184, 0, 4651, 8185, 2, 1, 8187, 4, 6, 0, 5, 5, 0, 5, 1, 0, 8187, 8186, 0, 0, 0, 8172, 5, 3521, 3, 4] payload_rows_per_dst=[16384, 16368, 16376, 16384, 16384, 16374, 16344, 7060] dup_rows_per_dst=[74, 18, 9272, 8, 20, 0, 0, 6] combine_rows_per_src=[16384, 16368, 16376, 16384, 16384, 16374, 16344, 7060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121640 payload_rows_remote=105256 dup_rows=9432 dup_rows_remote=9432 dedup_ratio=0.071960 router_tokens_per_expert=[3, 0, 8187, 3, 0, 1, 8176, 1, 5036, 8188, 1, 4, 8188, 2, 2, 0, 3, 3, 0, 3, 0, 0, 8188, 8180, 7, 0, 2, 8181, 0, 3173, 0, 4] payload_rows_per_dst=[16382, 16356, 14158, 16384, 16384, 16360, 16370, 9246] dup_rows_per_dst=[4, 0, 7170, 0, 10, 0, 10, 2238] combine_rows_per_src=[16382, 16356, 14158, 16384, 16384, 16360, 16370, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=879542272 dispatch_meta_bytes=458816 combine_payload_bytes=879542272 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=880001088 ingress_bytes=1231863808 link_bytes=1231863808 token_bytes=1759543360 weight_movement_bytes=352321536 total_nvlink_bytes=2111864896 weight_to_token_ratio=0.200235 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.897923120e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123068 payload_rows_remote=106684 dup_rows=8004 dup_rows_remote=8004 dedup_ratio=0.061066 router_tokens_per_expert=[0, 9, 8192, 0, 0, 0, 8121, 0, 4333, 8192, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 8119, 0, 3992, 0, 0] payload_rows_per_dst=[16384, 16242, 8670, 16384, 16382, 16384, 16238, 16384] dup_rows_per_dst=[18, 0, 2, 0, 0, 0, 0, 7984] combine_rows_per_src=[16384, 16242, 8670, 16384, 16382, 16384, 16238, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=869466112 dispatch_meta_bytes=458552 combine_payload_bytes=869466112 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=869924664 ingress_bytes=1221787648 link_bytes=1221787648 token_bytes=1739390776 weight_movement_bytes=352321536 total_nvlink_bytes=2091712312 weight_to_token_ratio=0.202555 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.841500830e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=862535680 dispatch_meta_bytes=458456 combine_payload_bytes=862535680 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=862994136 ingress_bytes=1567178752 link_bytes=1567178752 token_bytes=1725529816 weight_movement_bytes=704643072 total_nvlink_bytes=2430172888 weight_to_token_ratio=0.408363 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.775546839e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117674 payload_rows_remote=101290 dup_rows=13398 dup_rows_remote=13398 dedup_ratio=0.102219 router_tokens_per_expert=[3614, 0, 6, 6, 8191, 0, 0, 0, 8190, 2806, 8174, 954, 3211, 6, 5814, 8182, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8187, 0, 1] payload_rows_per_dst=[8868, 16382, 12964, 13954, 16384, 16380, 16366, 16376] dup_rows_per_dst=[3070, 0, 6218, 4108, 0, 0, 2, 0] combine_rows_per_src=[8868, 16382, 12964, 13954, 16384, 16380, 16366, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=862257152 dispatch_meta_bytes=458752 combine_payload_bytes=862257152 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=862715904 ingress_bytes=1214578688 link_bytes=1214578688 token_bytes=1724973056 weight_movement_bytes=352321536 total_nvlink_bytes=2077294592 weight_to_token_ratio=0.204248 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.801133663e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122458 payload_rows_remote=106074 dup_rows=8614 dup_rows_remote=1138 dedup_ratio=0.065720 router_tokens_per_expert=[5, 12, 8192, 0, 4, 0, 8178, 0, 4467, 8192, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 1, 0, 8174, 0, 3738, 0, 0] payload_rows_per_dst=[16112, 16358, 8144, 16376, 16354, 16378, 16352, 16384] dup_rows_per_dst=[34, 278, 0, 0, 0, 46, 780, 7476] combine_rows_per_src=[16112, 16358, 8144, 16376, 16354, 16378, 16352, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122600 payload_rows_remote=106216 dup_rows=8472 dup_rows_remote=40 dedup_ratio=0.064636 router_tokens_per_expert=[0, 20, 8191, 0, 0, 1, 8164, 0, 4216, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8135, 0, 4041, 0, 0] payload_rows_per_dst=[16384, 16328, 16384, 16384, 16384, 16384, 16270, 8082] dup_rows_per_dst=[38, 2, 8432, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16328, 16384, 16384, 16384, 16384, 16270, 8082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=873955328 dispatch_meta_bytes=458752 combine_payload_bytes=873955328 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=874414080 ingress_bytes=1578598400 link_bytes=1578598400 token_bytes=1748369408 weight_movement_bytes=704643072 total_nvlink_bytes=2453012480 weight_to_token_ratio=0.403029 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.839492101e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=829767680 dispatch_meta_bytes=458752 combine_payload_bytes=829767680 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=830226432 ingress_bytes=1182089216 link_bytes=1182089216 token_bytes=1659994112 weight_movement_bytes=352321536 total_nvlink_bytes=2012315648 weight_to_token_ratio=0.212243 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.619206181e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=868958208 dispatch_meta_bytes=428848 combine_payload_bytes=868958208 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=869387056 ingress_bytes=1221279744 link_bytes=1221279744 token_bytes=1738345264 weight_movement_bytes=352321536 total_nvlink_bytes=2090666800 weight_to_token_ratio=0.202676 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.838656779e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121044 payload_rows_remote=104660 dup_rows=10028 dup_rows_remote=10 dedup_ratio=0.076508 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6073, 5824, 8186, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 5, 0, 0, 7675, 11, 8192, 30, 4968, 0, 0, 8192, 0] payload_rows_per_dst=[16372, 12146, 11648, 16384, 16376, 15350, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 10, 10018, 0] combine_rows_per_src=[16372, 12146, 11648, 16384, 16376, 15350, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=870121472 dispatch_meta_bytes=425024 combine_payload_bytes=870121472 prefetch_bytes=176160768 reduce_bytes=1233125376 egress_bytes=870546496 ingress_bytes=2279407616 link_bytes=2279407616 token_bytes=1740667968 weight_movement_bytes=1409286144 total_nvlink_bytes=3149954112 weight_to_token_ratio=0.809624 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=18432 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.276373118e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120172 payload_rows_remote=105648 dup_rows=10900 dup_rows_remote=6736 dedup_ratio=0.083160 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6221, 5723, 8188, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8172, 15, 0, 0, 7948, 23, 8192, 18, 4646, 0, 0, 8192, 6] payload_rows_per_dst=[16376, 14524, 14892, 16384, 16344, 15896, 9372, 16384] dup_rows_per_dst=[0, 4164, 6692, 0, 0, 30, 2, 12] combine_rows_per_src=[16376, 14524, 14892, 16384, 16344, 15896, 9372, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 2, 0, 2, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112372 payload_rows_remote=95988 dup_rows=18700 dup_rows_remote=18670 dedup_ratio=0.142670 router_tokens_per_expert=[10, 0, 0, 5, 0, 10, 0, 7498, 5093, 8187, 5, 0, 5, 0, 6, 0, 8183, 10, 6, 8183, 10, 4, 0, 7766, 19, 8180, 113, 4082, 0, 5, 8156, 0] payload_rows_per_dst=[16384, 16374, 10186, 16378, 16374, 15786, 4570, 16320] dup_rows_per_dst=[30, 15002, 0, 12, 22, 3424, 210, 0] combine_rows_per_src=[16384, 16374, 10186, 16378, 16374, 15786, 4570, 16320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120346 payload_rows_remote=103962 dup_rows=10726 dup_rows_remote=10726 dedup_ratio=0.081833 router_tokens_per_expert=[7, 1, 0, 3, 3, 6, 1, 6333, 5046, 8186, 3, 0, 1, 0, 2, 0, 8189, 0, 4, 8186, 2, 3, 0, 7589, 1, 8188, 180, 5410, 0, 3, 8183, 6] payload_rows_per_dst=[16382, 12678, 15460, 16384, 16380, 15580, 11108, 16374] dup_rows_per_dst=[12, 8, 6728, 0, 152, 3254, 562, 10] combine_rows_per_src=[16382, 12678, 15460, 16384, 16380, 15580, 11108, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=857374720 dispatch_meta_bytes=418680 combine_payload_bytes=857374720 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=857793400 ingress_bytes=2618982400 link_bytes=2618982400 token_bytes=1715168120 weight_movement_bytes=1761607680 total_nvlink_bytes=3476775800 weight_to_token_ratio=1.027076 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=16640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.466520822e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=865468416 dispatch_meta_bytes=449536 combine_payload_bytes=865468416 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=865917952 ingress_bytes=1217789952 link_bytes=1217789952 token_bytes=1731386368 weight_movement_bytes=352321536 total_nvlink_bytes=2083707904 weight_to_token_ratio=0.203491 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=27392 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.819115401e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122932 payload_rows_remote=107364 dup_rows=8140 dup_rows_remote=8140 dedup_ratio=0.062103 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7050, 5698, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8166, 0, 0, 0, 7784, 0, 8192, 18, 4048, 0, 0, 8192, 4] payload_rows_per_dst=[16384, 14100, 11396, 16384, 16332, 15568, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8132, 8] combine_rows_per_src=[16384, 14100, 11396, 16384, 16332, 15568, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=786333696 dispatch_meta_bytes=458632 combine_payload_bytes=786333696 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=786792328 ingress_bytes=1314816000 link_bytes=1314816000 token_bytes=1573126024 weight_movement_bytes=528482304 total_nvlink_bytes=2101608328 weight_to_token_ratio=0.335944 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.362420769e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=851656704 dispatch_meta_bytes=458752 combine_payload_bytes=851656704 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=852115456 ingress_bytes=1380139008 link_bytes=1380139008 token_bytes=1703772160 weight_movement_bytes=528482304 total_nvlink_bytes=2232254464 weight_to_token_ratio=0.310184 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.728202346e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124820 payload_rows_remote=108436 dup_rows=6252 dup_rows_remote=6218 dedup_ratio=0.047699 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6844, 6686, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8172, 18, 0, 0, 7954, 1, 8192, 2, 3088, 0, 0, 8192, 4] payload_rows_per_dst=[16376, 13688, 13372, 16378, 16344, 15908, 16370, 16384] dup_rows_per_dst=[0, 0, 8, 0, 4, 36, 6170, 34] combine_rows_per_src=[16376, 13688, 13372, 16378, 16344, 15908, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122830 payload_rows_remote=106446 dup_rows=8242 dup_rows_remote=8242 dedup_ratio=0.062881 router_tokens_per_expert=[3, 15, 8190, 0, 3, 0, 8185, 0, 4253, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8017, 0, 4100, 0, 2] payload_rows_per_dst=[16384, 16370, 8506, 16384, 16384, 16384, 16034, 16384] dup_rows_per_dst=[32, 6, 0, 0, 0, 0, 0, 8204] combine_rows_per_src=[16384, 16370, 8506, 16384, 16384, 16384, 16034, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119162 payload_rows_remote=102778 dup_rows=11910 dup_rows_remote=26 dedup_ratio=0.090866 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7228, 5942, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8187, 2, 0, 0, 7490, 17, 8192, 11, 3894, 0, 0, 8191, 1] payload_rows_per_dst=[16378, 14456, 16384, 16384, 16374, 14980, 7824, 16382] dup_rows_per_dst=[0, 0, 11884, 0, 0, 4, 20, 2] combine_rows_per_src=[16378, 14456, 16384, 16384, 16374, 14980, 7824, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=879525888 dispatch_meta_bytes=462016 combine_payload_bytes=879525888 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=879987904 ingress_bytes=1231847424 link_bytes=1231847424 token_bytes=1759513792 weight_movement_bytes=352321536 total_nvlink_bytes=2111835328 weight_to_token_ratio=0.200238 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.897831376e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[3, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130824 payload_rows_remote=114440 dup_rows=248 dup_rows_remote=244 dedup_ratio=0.001892 router_tokens_per_expert=[0, 0, 0, 1, 8103, 8135, 8192, 0, 0, 4, 3, 78, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 8187, 0, 2, 8192, 0, 64, 0, 8189, 3] payload_rows_per_dst=[16382, 16206, 16326, 16384, 16384, 16374, 16384, 16384] dup_rows_per_dst=[2, 0, 114, 0, 0, 0, 4, 128] combine_rows_per_src=[16382, 16206, 16326, 16384, 16384, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=888307712 dispatch_meta_bytes=458616 combine_payload_bytes=888307712 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=888766328 ingress_bytes=1240629248 link_bytes=1240629248 token_bytes=1777074040 weight_movement_bytes=352321536 total_nvlink_bytes=2129395576 weight_to_token_ratio=0.198259 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.947005925e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=841957376 dispatch_meta_bytes=411216 combine_payload_bytes=841957376 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=842368592 ingress_bytes=1722761216 link_bytes=1722761216 token_bytes=1684325968 weight_movement_bytes=880803840 total_nvlink_bytes=2565129808 weight_to_token_ratio=0.522941 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.646743694e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130922 payload_rows_remote=114644 dup_rows=150 dup_rows_remote=150 dedup_ratio=0.001144 router_tokens_per_expert=[0, 0, 0, 1, 8139, 8142, 8192, 0, 0, 0, 2, 55, 0, 0, 0, 8192, 3, 0, 3, 0, 0, 0, 8189, 8181, 0, 7, 8192, 0, 41, 0, 8186, 11] payload_rows_per_dst=[16378, 16278, 16368, 16384, 16384, 16362, 16384, 16384] dup_rows_per_dst=[2, 0, 30, 0, 12, 0, 14, 92] combine_rows_per_src=[16378, 16278, 16368, 16384, 16384, 16362, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=872005632 dispatch_meta_bytes=458752 combine_payload_bytes=872005632 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=872464384 ingress_bytes=1224327168 link_bytes=1224327168 token_bytes=1744470016 weight_movement_bytes=352321536 total_nvlink_bytes=2096791552 weight_to_token_ratio=0.201965 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.855721081e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128722 payload_rows_remote=112348 dup_rows=2350 dup_rows_remote=2290 dedup_ratio=0.017929 router_tokens_per_expert=[5, 3, 7, 0, 7899, 8160, 8186, 3, 1, 1, 0, 146, 5, 7, 6, 8186, 0, 0, 1, 0, 5, 84, 8180, 8111, 4, 0, 8185, 0, 170, 6, 8166, 9] payload_rows_per_dst=[16374, 16096, 15930, 16384, 16362, 16336, 16352, 14888] dup_rows_per_dst=[60, 1800, 120, 24, 0, 46, 0, 300] combine_rows_per_src=[16374, 16096, 15930, 16384, 16362, 16336, 16352, 14888] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=937492480 dispatch_meta_bytes=458736 combine_payload_bytes=937492480 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=937951216 ingress_bytes=1642135552 link_bytes=1642135552 token_bytes=1875443696 weight_movement_bytes=704643072 total_nvlink_bytes=2580086768 weight_to_token_ratio=0.375721 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195273630e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130324 payload_rows_remote=113942 dup_rows=748 dup_rows_remote=742 dedup_ratio=0.005707 router_tokens_per_expert=[3, 3, 4, 0, 7878, 8105, 8189, 3, 7, 1, 9, 167, 3, 0, 3, 8188, 0, 0, 1, 0, 3, 72, 8162, 8171, 1, 1, 8187, 0, 184, 2, 8188, 1] payload_rows_per_dst=[16332, 15762, 16326, 16382, 16378, 16384, 16378, 16382] dup_rows_per_dst=[12, 0, 252, 6, 2, 108, 0, 368] combine_rows_per_src=[16332, 15762, 16326, 16382, 16378, 16384, 16378, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=459176 combine_payload_bytes=939163648 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=939622824 ingress_bytes=2172289024 link_bytes=2172289024 token_bytes=1878786472 weight_movement_bytes=1233125376 total_nvlink_bytes=3111911848 weight_to_token_ratio=0.656341 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216391178e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 6, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130950 payload_rows_remote=114578 dup_rows=122 dup_rows_remote=120 dedup_ratio=0.000931 router_tokens_per_expert=[0, 0, 0, 1, 8164, 8162, 8192, 0, 0, 0, 0, 13, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 1, 8191, 8186, 0, 0, 8192, 0, 50, 0, 8191, 0] payload_rows_per_dst=[16382, 16328, 16334, 16384, 16384, 16372, 16384, 16382] dup_rows_per_dst=[2, 0, 16, 0, 2, 2, 0, 100] combine_rows_per_src=[16382, 16328, 16334, 16384, 16384, 16372, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=920354816 dispatch_meta_bytes=458552 combine_payload_bytes=920354816 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=920813368 ingress_bytes=1448837120 link_bytes=1448837120 token_bytes=1841168184 weight_movement_bytes=528482304 total_nvlink_bytes=2369650488 weight_to_token_ratio=0.287036 gemm_rows_defined=66688 gemm_rows_charged=67584 gemm_tail_pad_rows=896 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.013258 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.112883098e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[3, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130796 payload_rows_remote=114412 dup_rows=276 dup_rows_remote=182 dedup_ratio=0.002106 router_tokens_per_expert=[0, 0, 0, 0, 8057, 8172, 8192, 0, 4, 9, 2, 52, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 8190, 0, 1, 8192, 0, 88, 0, 8190, 4] payload_rows_per_dst=[16382, 16114, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 94, 0, 0, 0, 2, 180] combine_rows_per_src=[16382, 16114, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130850 payload_rows_remote=114470 dup_rows=222 dup_rows_remote=46 dedup_ratio=0.001694 router_tokens_per_expert=[0, 0, 0, 1, 8132, 8189, 8192, 0, 0, 4, 2, 3, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 8191, 8188, 0, 3, 8192, 0, 57, 0, 8181, 8] payload_rows_per_dst=[16376, 16262, 16378, 16356, 16384, 16330, 16384, 16380] dup_rows_per_dst=[2, 0, 18, 0, 4, 6, 16, 176] combine_rows_per_src=[16376, 16262, 16378, 16356, 16384, 16330, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=933412864 dispatch_meta_bytes=458736 combine_payload_bytes=933412864 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=933871600 ingress_bytes=1461895168 link_bytes=1461895168 token_bytes=1867284464 weight_movement_bytes=528482304 total_nvlink_bytes=2395766768 weight_to_token_ratio=0.283022 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.186002716e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123486 payload_rows_remote=107116 dup_rows=7586 dup_rows_remote=7586 dedup_ratio=0.057877 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6966, 6211, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8185, 6, 0, 0, 7617, 6, 8192, 48, 3727, 0, 0, 8191, 6] payload_rows_per_dst=[16378, 13932, 12422, 16384, 16370, 15234, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 0, 0, 12, 7562, 12] combine_rows_per_src=[16378, 13932, 12422, 16384, 16370, 15234, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938622976 dispatch_meta_bytes=458792 combine_payload_bytes=938622976 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939081768 ingress_bytes=1995587584 link_bytes=1995587584 token_bytes=1877704744 weight_movement_bytes=1056964608 total_nvlink_bytes=2934669352 weight_to_token_ratio=0.562902 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117445747e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129918 payload_rows_remote=113796 dup_rows=1154 dup_rows_remote=1154 dedup_ratio=0.008804 router_tokens_per_expert=[519, 9, 8190, 0, 8192, 8002, 47, 0, 0, 1, 0, 0, 1, 2, 7, 8191, 0, 7930, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8061, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16016, 16384, 16384, 15860, 16384, 16122, 16384] dup_rows_per_dst=[1052, 82, 2, 18, 0, 0, 0, 0] combine_rows_per_src=[16384, 16016, 16384, 16384, 15860, 16384, 16122, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 3, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129768 payload_rows_remote=113938 dup_rows=1304 dup_rows_remote=942 dedup_ratio=0.009949 router_tokens_per_expert=[420, 22, 8185, 1, 8191, 7699, 397, 0, 0, 1, 0, 0, 2, 7, 25, 8191, 0, 7980, 0, 0, 8192, 0, 0, 0, 0, 0, 8189, 8034, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15830, 16378, 16382, 15960, 16384, 16068, 16382] dup_rows_per_dst=[872, 362, 2, 68, 0, 0, 0, 0] combine_rows_per_src=[16384, 15830, 16378, 16382, 15960, 16384, 16068, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=937263104 dispatch_meta_bytes=458376 combine_payload_bytes=937263104 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937721480 ingress_bytes=1289584640 link_bytes=1289584640 token_bytes=1874984584 weight_movement_bytes=352321536 total_nvlink_bytes=2227306120 weight_to_token_ratio=0.187906 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.221135685e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=937738240 dispatch_meta_bytes=458064 combine_payload_bytes=937738240 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=938196304 ingress_bytes=1642381312 link_bytes=1642381312 token_bytes=1875934544 weight_movement_bytes=704643072 total_nvlink_bytes=2580577616 weight_to_token_ratio=0.375622 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.196649784e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=877494272 dispatch_meta_bytes=458808 combine_payload_bytes=877494272 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=877953080 ingress_bytes=1758298112 link_bytes=1758298112 token_bytes=1755447352 weight_movement_bytes=880803840 total_nvlink_bytes=2636251192 weight_to_token_ratio=0.501755 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.845735477e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=932216832 dispatch_meta_bytes=459800 combine_payload_bytes=932216832 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=932676632 ingress_bytes=1636859904 link_bytes=1636859904 token_bytes=1864893464 weight_movement_bytes=704643072 total_nvlink_bytes=2569536536 weight_to_token_ratio=0.377846 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.165732203e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130238 payload_rows_remote=113864 dup_rows=834 dup_rows_remote=232 dedup_ratio=0.006363 router_tokens_per_expert=[282, 12, 8180, 0, 8186, 7997, 160, 5, 6, 57, 0, 5, 0, 0, 8, 8151, 5, 7924, 7, 5, 8185, 0, 1, 11, 0, 5, 8186, 8152, 0, 6, 0, 0] payload_rows_per_dst=[16374, 16302, 16342, 16286, 15862, 16374, 16316, 16382] dup_rows_per_dst=[602, 64, 124, 16, 20, 8, 0, 0] combine_rows_per_src=[16374, 16302, 16342, 16286, 15862, 16374, 16316, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=933380096 dispatch_meta_bytes=459520 combine_payload_bytes=933380096 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=933839616 ingress_bytes=1814183936 link_bytes=1814183936 token_bytes=1867219712 weight_movement_bytes=880803840 total_nvlink_bytes=2748023552 weight_to_token_ratio=0.471719 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.015867276e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 2, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129928 payload_rows_remote=113708 dup_rows=1144 dup_rows_remote=1140 dedup_ratio=0.008728 router_tokens_per_expert=[462, 6, 8186, 0, 8185, 7991, 142, 3, 4, 20, 0, 3, 0, 1, 2, 8109, 3, 7861, 6, 3, 8188, 0, 0, 3, 1, 0, 8188, 8162, 0, 4, 0, 3] payload_rows_per_dst=[16378, 16126, 16384, 16220, 15734, 16376, 16326, 16384] dup_rows_per_dst=[930, 146, 40, 4, 12, 6, 0, 6] combine_rows_per_src=[16378, 16126, 16384, 16220, 15734, 16376, 16326, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129490 payload_rows_remote=113106 dup_rows=1582 dup_rows_remote=1580 dedup_ratio=0.012070 router_tokens_per_expert=[726, 0, 8192, 0, 8192, 7948, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7804, 0, 0, 8192, 0, 0, 1, 0, 0, 8192, 8008, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15948, 16384, 16382, 15608, 16384, 16016, 16384] dup_rows_per_dst=[1452, 128, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 15948, 16384, 16382, 15608, 16384, 16016, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=932773888 dispatch_meta_bytes=456384 combine_payload_bytes=932773888 prefetch_bytes=176160768 reduce_bytes=1409286144 egress_bytes=933230272 ingress_bytes=2518220800 link_bytes=2518220800 token_bytes=1866004160 weight_movement_bytes=1585446912 total_nvlink_bytes=3451451072 weight_to_token_ratio=0.849648 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.410098532e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129926 payload_rows_remote=113542 dup_rows=1146 dup_rows_remote=1146 dedup_ratio=0.008743 router_tokens_per_expert=[500, 4, 8190, 0, 8192, 8035, 106, 0, 0, 0, 0, 0, 0, 1, 6, 8191, 0, 7895, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8032, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16154, 16384, 16382, 15790, 16384, 16064, 16384] dup_rows_per_dst=[1004, 128, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[16384, 16154, 16384, 16382, 15790, 16384, 16064, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=931495936 dispatch_meta_bytes=459392 combine_payload_bytes=931495936 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=931955328 ingress_bytes=1459978240 link_bytes=1459978240 token_bytes=1863451264 weight_movement_bytes=528482304 total_nvlink_bytes=2391933568 weight_to_token_ratio=0.283604 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.175268719e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122682 payload_rows_remote=106302 dup_rows=8390 dup_rows_remote=8388 dedup_ratio=0.064011 router_tokens_per_expert=[521, 13, 8186, 0, 8192, 7731, 252, 0, 0, 0, 0, 0, 1, 4, 13, 8191, 0, 8016, 0, 0, 8192, 0, 0, 0, 0, 0, 8188, 8036, 0, 0, 0, 0] payload_rows_per_dst=[9158, 15868, 16270, 16382, 16152, 16384, 16088, 16380] dup_rows_per_dst=[454, 1620, 0, 120, 4372, 2, 1820, 2] combine_rows_per_src=[9158, 15868, 16270, 16382, 16152, 16384, 16088, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131046 payload_rows_remote=114662 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000198 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 9, 4, 0, 8192, 8185, 0, 8192, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8186, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16372, 16382, 16384, 16372] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16372, 16382, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=926564352 dispatch_meta_bytes=458744 combine_payload_bytes=926564352 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=927023096 ingress_bytes=1278885888 link_bytes=1278885888 token_bytes=1853587448 weight_movement_bytes=352321536 total_nvlink_bytes=2205908984 weight_to_token_ratio=0.190075 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.161227140e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130886 payload_rows_remote=114502 dup_rows=186 dup_rows_remote=178 dedup_ratio=0.001419 router_tokens_per_expert=[0, 0, 0, 3, 8113, 8145, 8192, 0, 0, 3, 1, 42, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 8190, 8186, 0, 1, 8192, 0, 75, 0, 8187, 10] payload_rows_per_dst=[16380, 16226, 16372, 16384, 16384, 16372, 16384, 16384] dup_rows_per_dst=[6, 0, 10, 0, 8, 0, 2, 160] combine_rows_per_src=[16380, 16226, 16372, 16384, 16384, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131028 payload_rows_remote=114644 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 15, 7, 0, 8192, 8185, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8183, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16370, 16372, 16384, 16366] dup_rows_per_dst=[0, 0, 0, 44, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16370, 16372, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 6, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=930136064 dispatch_meta_bytes=458752 combine_payload_bytes=930136064 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930594816 ingress_bytes=1282457600 link_bytes=1282457600 token_bytes=1860730880 weight_movement_bytes=352321536 total_nvlink_bytes=2213052416 weight_to_token_ratio=0.189346 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.181227236e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=870825984 dispatch_meta_bytes=458760 combine_payload_bytes=870825984 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=871284744 ingress_bytes=1399308288 link_bytes=1399308288 token_bytes=1742110728 weight_movement_bytes=528482304 total_nvlink_bytes=2270593032 weight_to_token_ratio=0.303357 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.835542313e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=458752 combine_payload_bytes=939311104 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=939769856 ingress_bytes=1643954176 link_bytes=1643954176 token_bytes=1879080960 weight_movement_bytes=704643072 total_nvlink_bytes=2583724032 weight_to_token_ratio=0.374993 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205457165e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938000384 dispatch_meta_bytes=458720 combine_payload_bytes=938000384 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=938459104 ingress_bytes=1466482688 link_bytes=1466482688 token_bytes=1876459488 weight_movement_bytes=528482304 total_nvlink_bytes=2404941792 weight_to_token_ratio=0.281638 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.211690913e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=458752 combine_payload_bytes=939163648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939622400 ingress_bytes=1291485184 link_bytes=1291485184 token_bytes=1878786048 weight_movement_bytes=352321536 total_nvlink_bytes=2231107584 weight_to_token_ratio=0.187526 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231777938e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130894 payload_rows_remote=114512 dup_rows=178 dup_rows_remote=120 dedup_ratio=0.001358 router_tokens_per_expert=[5, 30, 0, 8185, 8156, 6, 0, 6, 12, 5, 8181, 5, 3, 0, 6, 8180, 8183, 0, 8180, 0, 6, 7, 8186, 0, 9, 0, 0, 0, 0, 8181, 4, 0] payload_rows_per_dst=[16382, 16334, 16384, 16300, 16366, 16382, 16378, 16368] dup_rows_per_dst=[58, 60, 44, 0, 0, 14, 0, 2] combine_rows_per_src=[16382, 16334, 16384, 16300, 16366, 16382, 16378, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130972 payload_rows_remote=114592 dup_rows=100 dup_rows_remote=90 dedup_ratio=0.000763 router_tokens_per_expert=[3, 24, 0, 8188, 8157, 3, 0, 3, 7, 12, 8183, 3, 4, 1, 4, 8186, 8188, 0, 8184, 1, 4, 4, 8188, 1, 2, 0, 0, 0, 0, 8183, 3, 0] payload_rows_per_dst=[16382, 16326, 16382, 16380, 16376, 16382, 16372, 16372] dup_rows_per_dst=[48, 0, 28, 10, 2, 12, 0, 0] combine_rows_per_src=[16382, 16326, 16382, 16380, 16376, 16382, 16372, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114682 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000046 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 1, 2, 0, 8192, 8189, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938082304 dispatch_meta_bytes=458528 combine_payload_bytes=938082304 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=938540832 ingress_bytes=1818886144 link_bytes=1818886144 token_bytes=1876623136 weight_movement_bytes=880803840 total_nvlink_bytes=2757426976 weight_to_token_ratio=0.469356 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018500317e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114684 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8190, 0, 2, 1, 0, 8192, 8192, 0, 8192, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16382, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 6, 2, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16382, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938737664 dispatch_meta_bytes=458728 combine_payload_bytes=938737664 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=939196392 ingress_bytes=1819541504 link_bytes=1819541504 token_bytes=1877934056 weight_movement_bytes=880803840 total_nvlink_bytes=2758737896 weight_to_token_ratio=0.469028 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018867291e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129374 payload_rows_remote=112990 dup_rows=1698 dup_rows_remote=172 dedup_ratio=0.012955 router_tokens_per_expert=[0, 0, 0, 8189, 78, 6, 0, 0, 8192, 8191, 0, 7385, 0, 0, 8152, 2, 8192, 0, 0, 8192, 8192, 0, 0, 0, 750, 0, 13, 0, 0, 0, 2, 0] payload_rows_per_dst=[16378, 16384, 14770, 16308, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 168, 0, 0, 0, 0, 1526, 4] combine_rows_per_src=[16378, 16384, 14770, 16308, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130006 payload_rows_remote=114304 dup_rows=1066 dup_rows_remote=1066 dedup_ratio=0.008133 router_tokens_per_expert=[430, 8, 8192, 0, 8192, 7971, 178, 0, 0, 0, 0, 0, 0, 3, 12, 8192, 0, 7851, 0, 0, 8192, 0, 0, 0, 0, 0, 8189, 8126, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16138, 16378, 16384, 15702, 16384, 16252, 16384] dup_rows_per_dst=[876, 160, 0, 30, 0, 0, 0, 0] combine_rows_per_src=[16384, 16138, 16378, 16384, 15702, 16384, 16252, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=458752 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939933696 ingress_bytes=1291796480 link_bytes=1291796480 token_bytes=1879408640 weight_movement_bytes=352321536 total_nvlink_bytes=2231730176 weight_to_token_ratio=0.187464 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233521066e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130830 payload_rows_remote=114460 dup_rows=242 dup_rows_remote=158 dedup_ratio=0.001846 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 8, 3, 0, 8192, 8189, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8185, 0, 0] payload_rows_per_dst=[16282, 16384, 16314, 16336, 16378, 16382, 16384, 16370] dup_rows_per_dst=[0, 48, 4, 22, 62, 4, 18, 84] combine_rows_per_src=[16282, 16384, 16314, 16336, 16378, 16382, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120674 payload_rows_remote=104290 dup_rows=10398 dup_rows_remote=10252 dedup_ratio=0.079330 router_tokens_per_expert=[0, 0, 0, 8184, 57, 16, 0, 0, 8192, 8189, 0, 7518, 0, 0, 8064, 4, 8192, 0, 0, 8192, 8192, 0, 0, 0, 719, 0, 14, 0, 0, 0, 3, 0] payload_rows_per_dst=[16368, 16384, 16110, 16136, 16384, 16384, 6524, 16384] dup_rows_per_dst=[0, 146, 10050, 0, 0, 0, 202, 0] combine_rows_per_src=[16368, 16384, 16110, 16136, 16384, 16384, 6524, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458768 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939950096 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=1879441424 weight_movement_bytes=704643072 total_nvlink_bytes=2584084496 weight_to_token_ratio=0.374922 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=925614080 dispatch_meta_bytes=452648 combine_payload_bytes=925614080 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=926066728 ingress_bytes=1277935616 link_bytes=1277935616 token_bytes=1851680808 weight_movement_bytes=352321536 total_nvlink_bytes=2204002344 weight_to_token_ratio=0.190271 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=27008 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.155906013e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=936378368 dispatch_meta_bytes=461480 combine_payload_bytes=936378368 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936839848 ingress_bytes=1288699904 link_bytes=1288699904 token_bytes=1873218216 weight_movement_bytes=352321536 total_nvlink_bytes=2225539752 weight_to_token_ratio=0.188084 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.216181533e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=937656320 dispatch_meta_bytes=458472 combine_payload_bytes=937656320 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938114792 ingress_bytes=1289977856 link_bytes=1289977856 token_bytes=1875771112 weight_movement_bytes=352321536 total_nvlink_bytes=2228092648 weight_to_token_ratio=0.187828 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223337530e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=854343680 dispatch_meta_bytes=458168 combine_payload_bytes=854343680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=854801848 ingress_bytes=1206665216 link_bytes=1206665216 token_bytes=1709145528 weight_movement_bytes=352321536 total_nvlink_bytes=2061467064 weight_to_token_ratio=0.206139 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.756821523e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125620 payload_rows_remote=109398 dup_rows=5452 dup_rows_remote=5452 dedup_ratio=0.041595 router_tokens_per_expert=[3, 4, 141, 8185, 202, 3, 0, 3, 8183, 8188, 4, 5572, 2, 0, 8108, 1, 8188, 0, 4, 8187, 8166, 1, 0, 4, 2378, 0, 0, 4, 0, 0, 1, 4] payload_rows_per_dst=[16382, 16382, 11152, 16222, 16382, 16342, 16374, 16384] dup_rows_per_dst=[284, 410, 0, 0, 0, 0, 4756, 2] combine_rows_per_src=[16382, 16382, 11152, 16222, 16382, 16342, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119508 payload_rows_remote=103124 dup_rows=11564 dup_rows_remote=10920 dedup_ratio=0.088226 router_tokens_per_expert=[5, 7, 202, 8185, 294, 0, 0, 6, 8177, 8183, 7, 4525, 1, 0, 7985, 0, 8184, 0, 6, 8183, 8141, 6, 0, 7, 3417, 0, 1, 8, 0, 0, 0, 6] payload_rows_per_dst=[16384, 16382, 16376, 16244, 14594, 16306, 6850, 16372] dup_rows_per_dst=[644, 586, 9048, 1282, 0, 2, 2, 0] combine_rows_per_src=[16384, 16382, 16376, 16244, 14594, 16306, 6850, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129900 payload_rows_remote=113516 dup_rows=1172 dup_rows_remote=1172 dedup_ratio=0.008942 router_tokens_per_expert=[0, 0, 1, 8192, 46, 0, 0, 0, 8192, 8192, 0, 7650, 0, 0, 8148, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 538, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15300, 16296, 16384, 16384, 16384, 16384] dup_rows_per_dst=[2, 92, 0, 0, 0, 0, 1078, 0] combine_rows_per_src=[16384, 16384, 15300, 16296, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126074 payload_rows_remote=114622 dup_rows=4998 dup_rows_remote=4998 dedup_ratio=0.038132 router_tokens_per_expert=[0, 0, 0, 8191, 112, 4, 0, 0, 8192, 8192, 0, 5726, 0, 0, 8158, 2, 8192, 0, 0, 8192, 8192, 0, 0, 0, 2380, 0, 2, 0, 0, 0, 1, 0] payload_rows_per_dst=[16382, 16384, 11452, 16320, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 232, 0, 0, 0, 0, 4764, 2] combine_rows_per_src=[16382, 16384, 11452, 16320, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=896188416 dispatch_meta_bytes=459400 combine_payload_bytes=896188416 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=896647816 ingress_bytes=1248509952 link_bytes=1248509952 token_bytes=1792836232 weight_movement_bytes=352321536 total_nvlink_bytes=2145157768 weight_to_token_ratio=0.196516 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.991134577e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108962 payload_rows_remote=103666 dup_rows=22110 dup_rows_remote=22110 dedup_ratio=0.168686 router_tokens_per_expert=[0, 0, 1654, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 8192, 0, 0, 0, 2648, 0, 0, 0, 0, 8187, 0, 8162] payload_rows_per_dst=[13896, 16384, 16384, 16384, 7850, 16384, 5296, 16384] dup_rows_per_dst=[2432, 0, 56, 0, 0, 0, 0, 19622] combine_rows_per_src=[13896, 16384, 16384, 16384, 7850, 16384, 5296, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=844791808 dispatch_meta_bytes=456176 combine_payload_bytes=844791808 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=845247984 ingress_bytes=1549434880 link_bytes=1549434880 token_bytes=1690039792 weight_movement_bytes=704643072 total_nvlink_bytes=2394682864 weight_to_token_ratio=0.416939 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.676188562e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131050 payload_rows_remote=114668 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000168 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 7, 4, 0, 8192, 8191, 0, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8186, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16376, 16384, 16372] dup_rows_per_dst=[0, 0, 0, 22, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16376, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 3] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114672 payload_rows_remote=98290 dup_rows=16400 dup_rows_remote=16400 dedup_ratio=0.125122 router_tokens_per_expert=[0, 1, 1589, 0, 8191, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 8192, 0, 0, 0, 2846, 0, 0, 0, 0, 8179, 0, 8178] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16374, 16384, 16380, 0] dup_rows_per_dst=[3180, 0, 0, 0, 7544, 0, 5676, 0] combine_rows_per_src=[16384, 16382, 16384, 16384, 16374, 16384, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=929923072 dispatch_meta_bytes=458752 combine_payload_bytes=929923072 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930381824 ingress_bytes=1282244608 link_bytes=1282244608 token_bytes=1860304896 weight_movement_bytes=352321536 total_nvlink_bytes=2212626432 weight_to_token_ratio=0.189389 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.180034570e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=478480 combine_payload_bytes=938983424 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=939461904 ingress_bytes=2172108800 link_bytes=2172108800 token_bytes=1878445328 weight_movement_bytes=1233125376 total_nvlink_bytes=3111570704 weight_to_token_ratio=0.656461 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=26752 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216290260e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=849231872 dispatch_meta_bytes=503104 combine_payload_bytes=849231872 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=849734976 ingress_bytes=1201553408 link_bytes=1201553408 token_bytes=1698966848 weight_movement_bytes=352321536 total_nvlink_bytes=2051288384 weight_to_token_ratio=0.207374 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.728197532e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124270 payload_rows_remote=107888 dup_rows=6802 dup_rows_remote=6794 dedup_ratio=0.051895 router_tokens_per_expert=[0, 0, 0, 8192, 45, 3, 0, 0, 8192, 8190, 0, 6255, 0, 0, 8057, 8, 8192, 0, 0, 8192, 8192, 0, 0, 0, 2009, 0, 8, 0, 0, 0, 1, 0] payload_rows_per_dst=[15484, 14484, 12638, 16130, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 86, 764, 0, 1792, 118, 4034, 8] combine_rows_per_src=[15484, 14484, 12638, 16130, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458760 combine_payload_bytes=939360256 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939819016 ingress_bytes=1996324864 link_bytes=1996324864 token_bytes=1879179272 weight_movement_bytes=1056964608 total_nvlink_bytes=2936143880 weight_to_token_ratio=0.562461 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117858593e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=805191680 dispatch_meta_bytes=458760 combine_payload_bytes=805191680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=805650440 ingress_bytes=1157513216 link_bytes=1157513216 token_bytes=1610842120 weight_movement_bytes=352321536 total_nvlink_bytes=1963163656 weight_to_token_ratio=0.218719 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.481590840e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121892 payload_rows_remote=105508 dup_rows=9180 dup_rows_remote=9170 dedup_ratio=0.070038 router_tokens_per_expert=[0, 9, 902, 1, 8174, 0, 3, 4, 8188, 8187, 8187, 4, 4, 5, 0, 0, 0, 3, 0, 5356, 8187, 3, 5, 0, 2154, 22, 0, 0, 0, 8066, 3, 8069] payload_rows_per_dst=[16376, 16356, 16382, 16384, 14430, 16382, 16328, 9254] dup_rows_per_dst=[1822, 6, 0, 10, 3352, 8, 3982, 0] combine_rows_per_src=[16376, 16356, 16382, 16384, 14430, 16382, 16328, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=883818496 dispatch_meta_bytes=458728 combine_payload_bytes=883818496 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=884277224 ingress_bytes=1412300800 link_bytes=1412300800 token_bytes=1768095720 weight_movement_bytes=528482304 total_nvlink_bytes=2296578024 weight_to_token_ratio=0.298899 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.908294957e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114416 payload_rows_remote=98044 dup_rows=16656 dup_rows_remote=15466 dedup_ratio=0.127075 router_tokens_per_expert=[0, 3, 593, 3, 8143, 0, 6, 6, 8186, 8181, 8182, 7, 8, 5, 0, 0, 0, 5, 0, 6027, 8185, 5, 8, 0, 1937, 30, 0, 0, 0, 7892, 2, 8122] payload_rows_per_dst=[16372, 16298, 16258, 16364, 16370, 16370, 16380, 4] dup_rows_per_dst=[1190, 152, 18, 10, 11464, 10, 3812, 0] combine_rows_per_src=[16372, 16298, 16258, 16364, 16370, 16370, 16380, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119370 payload_rows_remote=102986 dup_rows=11702 dup_rows_remote=11702 dedup_ratio=0.089279 router_tokens_per_expert=[0, 0, 1054, 0, 8188, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5010, 8192, 0, 0, 0, 2257, 0, 0, 0, 0, 8190, 0, 8069] payload_rows_per_dst=[16384, 16376, 16384, 16384, 10020, 16384, 11086, 16352] dup_rows_per_dst=[2108, 0, 0, 0, 0, 0, 2830, 6764] combine_rows_per_src=[16384, 16376, 16384, 16384, 10020, 16384, 11086, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=864321536 dispatch_meta_bytes=458712 combine_payload_bytes=864321536 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=864780248 ingress_bytes=1216643072 link_bytes=1216643072 token_bytes=1729101784 weight_movement_bytes=352321536 total_nvlink_bytes=2081423320 weight_to_token_ratio=0.203760 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.812693351e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128360 payload_rows_remote=113920 dup_rows=2712 dup_rows_remote=2100 dedup_ratio=0.020691 router_tokens_per_expert=[0, 0, 2, 0, 1, 0, 0, 0, 1, 646, 6, 0, 0, 0, 0, 8142, 8172, 199, 0, 8192, 0, 7876, 8192, 0, 0, 0, 4602, 2924, 0, 8192, 197, 8192] payload_rows_per_dst=[16354, 16384, 16384, 16284, 16378, 15752, 14440, 16384] dup_rows_per_dst=[4, 2, 1306, 0, 394, 0, 612, 394] combine_rows_per_src=[16354, 16384, 16384, 16284, 16378, 15752, 14440, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114668 payload_rows_remote=98284 dup_rows=16404 dup_rows_remote=16404 dedup_ratio=0.125153 router_tokens_per_expert=[0, 0, 1418, 0, 8185, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4481, 8191, 0, 0, 0, 2322, 0, 0, 0, 0, 8186, 0, 8177] payload_rows_per_dst=[16384, 16370, 16384, 16384, 16384, 16382, 16380, 0] dup_rows_per_dst=[2836, 0, 0, 0, 8938, 0, 4630, 0] combine_rows_per_src=[16384, 16370, 16384, 16384, 16384, 16382, 16380, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127846 payload_rows_remote=111462 dup_rows=3226 dup_rows_remote=3226 dedup_ratio=0.024612 router_tokens_per_expert=[0, 0, 0, 8190, 81, 11, 0, 0, 8192, 8192, 0, 6706, 0, 0, 8067, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 1516, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 13412, 16134, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 184, 0, 0, 0, 0, 3042, 0] combine_rows_per_src=[16380, 16384, 13412, 16134, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128790 payload_rows_remote=112406 dup_rows=2282 dup_rows_remote=2280 dedup_ratio=0.017410 router_tokens_per_expert=[0, 0, 5, 0, 1, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 8048, 8181, 219, 0, 8192, 0, 7782, 8192, 0, 0, 0, 6221, 1865, 0, 8192, 230, 8192] payload_rows_per_dst=[16366, 16384, 16384, 16096, 16380, 15564, 15232, 16384] dup_rows_per_dst=[10, 2, 432, 0, 438, 0, 940, 460] combine_rows_per_src=[16366, 16384, 16384, 16096, 16380, 15564, 15232, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 2, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=803176448 dispatch_meta_bytes=454040 combine_payload_bytes=803176448 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=803630488 ingress_bytes=1155497984 link_bytes=1155497984 token_bytes=1606806936 weight_movement_bytes=352321536 total_nvlink_bytes=1959128472 weight_to_token_ratio=0.219268 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.470306382e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=843661312 dispatch_meta_bytes=458752 combine_payload_bytes=843661312 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=844120064 ingress_bytes=1195982848 link_bytes=1195982848 token_bytes=1687781376 weight_movement_bytes=352321536 total_nvlink_bytes=2040102912 weight_to_token_ratio=0.208748 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.697004721e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=933232640 dispatch_meta_bytes=464080 combine_payload_bytes=933232640 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933696720 ingress_bytes=1285554176 link_bytes=1285554176 token_bytes=1866929360 weight_movement_bytes=352321536 total_nvlink_bytes=2219250896 weight_to_token_ratio=0.188717 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.198566769e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=805142528 dispatch_meta_bytes=458752 combine_payload_bytes=805142528 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=805601280 ingress_bytes=2566750208 link_bytes=2566750208 token_bytes=1610743808 weight_movement_bytes=1761607680 total_nvlink_bytes=3372351488 weight_to_token_ratio=1.093661 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=29952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.437272975e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=913096704 dispatch_meta_bytes=458752 combine_payload_bytes=913096704 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=913555456 ingress_bytes=2322382848 link_bytes=2322382848 token_bytes=1826652160 weight_movement_bytes=1409286144 total_nvlink_bytes=3235938304 weight_to_token_ratio=0.771513 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.300437454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=920829952 dispatch_meta_bytes=458744 combine_payload_bytes=920829952 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=921288696 ingress_bytes=1449312256 link_bytes=1449312256 token_bytes=1842118648 weight_movement_bytes=528482304 total_nvlink_bytes=2370600952 weight_to_token_ratio=0.286888 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.115543661e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=100930 payload_rows_remote=84546 dup_rows=30142 dup_rows_remote=190 dedup_ratio=0.229965 router_tokens_per_expert=[0, 1, 854, 0, 8188, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3155, 8192, 0, 0, 0, 4322, 0, 0, 0, 0, 8182, 0, 8066] payload_rows_per_dst=[4064, 16376, 16384, 16384, 6310, 16384, 8644, 16384] dup_rows_per_dst=[136, 0, 54, 0, 0, 0, 0, 29952] combine_rows_per_src=[4064, 16376, 16384, 16384, 6310, 16384, 8644, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124184 payload_rows_remote=107888 dup_rows=6888 dup_rows_remote=6876 dedup_ratio=0.052551 router_tokens_per_expert=[0, 0, 6, 5, 1, 0, 1, 4, 0, 1188, 3, 4, 3, 3, 4, 8144, 7887, 234, 4, 8086, 24, 7454, 8175, 1, 4, 0, 2397, 3542, 3, 8058, 2114, 8187] payload_rows_per_dst=[16178, 16384, 16384, 16296, 16146, 14956, 11456, 16384] dup_rows_per_dst=[0, 2, 2356, 12, 120, 2, 430, 3966] combine_rows_per_src=[16178, 16384, 16384, 16296, 16146, 14956, 11456, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=692600832 dispatch_meta_bytes=338944 combine_payload_bytes=692600832 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=692939776 ingress_bytes=1749565440 link_bytes=1749565440 token_bytes=1385540608 weight_movement_bytes=1056964608 total_nvlink_bytes=2442505216 weight_to_token_ratio=0.762854 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.796836159e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=883818496 dispatch_meta_bytes=459056 combine_payload_bytes=883818496 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=884277552 ingress_bytes=1236140032 link_bytes=1236140032 token_bytes=1768096048 weight_movement_bytes=352321536 total_nvlink_bytes=2120417584 weight_to_token_ratio=0.199266 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.921868189e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114678 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 8192, 0, 8189, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8190] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16378, 16384, 16380] dup_rows_per_dst=[0, 0, 0, 10, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16378, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116252 payload_rows_remote=100174 dup_rows=14820 dup_rows_remote=14236 dedup_ratio=0.113068 router_tokens_per_expert=[4, 0, 1, 7, 0, 0, 1, 6, 0, 1313, 0, 7, 5, 5, 6, 8163, 7999, 231, 6, 7933, 40, 6825, 8053, 7, 7, 0, 3038, 3088, 0, 8018, 2590, 8183] payload_rows_per_dst=[16078, 16380, 16382, 16338, 14904, 13662, 16326, 6182] dup_rows_per_dst=[584, 2, 2364, 866, 32, 0, 10934, 38] combine_rows_per_src=[16078, 16380, 16382, 16338, 14904, 13662, 16326, 6182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128972 payload_rows_remote=113044 dup_rows=2100 dup_rows_remote=2100 dedup_ratio=0.016022 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 8180, 8181, 87, 0, 8192, 0, 7964, 8192, 0, 0, 0, 5010, 2606, 0, 8192, 280, 8192] payload_rows_per_dst=[16374, 16384, 16384, 16360, 16374, 15928, 14784, 16384] dup_rows_per_dst=[0, 0, 920, 0, 172, 0, 448, 560] combine_rows_per_src=[16374, 16384, 16384, 16360, 16374, 15928, 14784, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128454 payload_rows_remote=112070 dup_rows=2618 dup_rows_remote=1918 dedup_ratio=0.019974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 8138, 8181, 222, 0, 8192, 0, 7675, 8191, 0, 0, 0, 5707, 2249, 0, 8192, 246, 8192] payload_rows_per_dst=[16362, 16384, 16384, 16276, 16384, 15350, 14930, 16384] dup_rows_per_dst=[0, 0, 700, 0, 444, 0, 982, 492] combine_rows_per_src=[16362, 16384, 16384, 16276, 16384, 15350, 14930, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114678 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8187] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] dup_rows_per_dst=[0, 0, 0, 10, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126674 payload_rows_remote=114638 dup_rows=4398 dup_rows_remote=4398 dedup_ratio=0.033554 router_tokens_per_expert=[0, 1, 561, 0, 8186, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 6018, 8192, 0, 0, 0, 1641, 0, 0, 0, 0, 8186, 0, 8175] payload_rows_per_dst=[16384, 16372, 16384, 16384, 12036, 16384, 16374, 16356] dup_rows_per_dst=[1124, 0, 0, 0, 0, 0, 3274, 0] combine_rows_per_src=[16384, 16372, 16384, 16384, 12036, 16384, 16374, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458752 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939900928 ingress_bytes=1820246016 link_bytes=1820246016 token_bytes=1879343104 weight_movement_bytes=880803840 total_nvlink_bytes=2760146944 weight_to_token_ratio=0.468676 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019261788e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=820625408 dispatch_meta_bytes=457640 combine_payload_bytes=820625408 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=821083048 ingress_bytes=1525268480 link_bytes=1525268480 token_bytes=1641708456 weight_movement_bytes=704643072 total_nvlink_bytes=2346351528 weight_to_token_ratio=0.429213 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.540866810e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.375000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=926056448 dispatch_meta_bytes=460576 combine_payload_bytes=926056448 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=926517024 ingress_bytes=1983021056 link_bytes=1983021056 token_bytes=1852573472 weight_movement_bytes=1056964608 total_nvlink_bytes=2909538080 weight_to_token_ratio=0.570539 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=30080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.110409016e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=918077440 dispatch_meta_bytes=455952 combine_payload_bytes=918077440 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=918533392 ingress_bytes=1270398976 link_bytes=1270398976 token_bytes=1836610832 weight_movement_bytes=352321536 total_nvlink_bytes=2188932368 weight_to_token_ratio=0.191832 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=30080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.113703975e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458752 combine_payload_bytes=939442176 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=939900928 ingress_bytes=1467924480 link_bytes=1467924480 token_bytes=1879343104 weight_movement_bytes=528482304 total_nvlink_bytes=2407825408 weight_to_token_ratio=0.281206 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219764346e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130592 payload_rows_remote=114208 dup_rows=480 dup_rows_remote=32 dedup_ratio=0.003662 router_tokens_per_expert=[2, 4, 8180, 8142, 3, 0, 1, 0, 7, 0, 4, 5, 0, 45, 5, 186, 8185, 6, 3, 8187, 0, 8002, 3, 3, 8188, 1, 8182, 3, 0, 0, 3, 8186] payload_rows_per_dst=[16294, 16382, 16384, 16384, 16382, 16016, 16372, 16378] dup_rows_per_dst=[2, 0, 24, 448, 6, 0, 0, 0] combine_rows_per_src=[16294, 16382, 16384, 16384, 16382, 16016, 16372, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=476144 combine_payload_bytes=939114496 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939590640 ingress_bytes=1291436032 link_bytes=1291436032 token_bytes=1878705136 weight_movement_bytes=352321536 total_nvlink_bytes=2231026672 weight_to_token_ratio=0.187534 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=19968 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231502707e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117524 payload_rows_remote=101140 dup_rows=13548 dup_rows_remote=9484 dedup_ratio=0.103363 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 1085, 0, 0, 0, 0, 0, 8170, 8181, 106, 0, 8192, 0, 7941, 8192, 0, 0, 0, 2845, 3945, 0, 8192, 494, 8192] payload_rows_per_dst=[16374, 16378, 6410, 16340, 16172, 16330, 13136, 16384] dup_rows_per_dst=[0, 2, 840, 0, 200, 7998, 444, 4064] combine_rows_per_src=[16374, 16378, 6410, 16340, 16172, 16330, 13136, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 3, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131044 payload_rows_remote=114660 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000214 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 1, 0, 0, 8192, 2, 8192, 0, 9, 8192, 0, 0, 0, 8179, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16358, 16384, 16384, 16382] dup_rows_per_dst=[0, 2, 4, 18, 0, 4, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16358, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=935591936 dispatch_meta_bytes=456960 combine_payload_bytes=935591936 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936048896 ingress_bytes=1287913472 link_bytes=1287913472 token_bytes=1871640832 weight_movement_bytes=352321536 total_nvlink_bytes=2223962368 weight_to_token_ratio=0.188242 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.211777842e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129896 payload_rows_remote=113540 dup_rows=1176 dup_rows_remote=166 dedup_ratio=0.008972 router_tokens_per_expert=[5, 7, 8167, 8127, 5, 0, 2, 0, 0, 0, 8, 7, 0, 57, 8, 250, 8180, 9, 4, 8183, 0, 7933, 13, 3, 8185, 19, 8169, 5, 0, 0, 6, 8184] payload_rows_per_dst=[16356, 16384, 16384, 15740, 16374, 15898, 16380, 16380] dup_rows_per_dst=[1010, 2, 12, 140, 8, 0, 4, 0] combine_rows_per_src=[16356, 16384, 16384, 15740, 16374, 15898, 16380, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=828538880 dispatch_meta_bytes=442496 combine_payload_bytes=828538880 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=828981376 ingress_bytes=2061664256 link_bytes=2061664256 token_bytes=1657520256 weight_movement_bytes=1233125376 total_nvlink_bytes=2890645632 weight_to_token_ratio=0.743958 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.154445925e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 8191, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114688 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 8190, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131008 payload_rows_remote=114624 dup_rows=64 dup_rows_remote=64 dedup_ratio=0.000488 router_tokens_per_expert=[8187, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 9, 8192, 0, 19, 8192, 0, 0, 0, 8165, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16374, 16384, 16384, 16384, 16330, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 18, 38, 0, 8, 0, 0] combine_rows_per_src=[16374, 16384, 16384, 16384, 16330, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=458752 combine_payload_bytes=939294720 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939753472 ingress_bytes=1291616256 link_bytes=1291616256 token_bytes=1879048192 weight_movement_bytes=352321536 total_nvlink_bytes=2231369728 weight_to_token_ratio=0.187500 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232511886e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128734 payload_rows_remote=112362 dup_rows=2338 dup_rows_remote=2176 dedup_ratio=0.017838 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 8102, 8171, 96, 0, 8192, 0, 7765, 8190, 0, 0, 0, 4598, 3391, 0, 8192, 114, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16204, 16372, 15530, 15092, 16384] dup_rows_per_dst=[4, 0, 1058, 0, 162, 0, 886, 228] combine_rows_per_src=[16384, 16384, 16384, 16204, 16372, 15530, 15092, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=930119680 dispatch_meta_bytes=454824 combine_payload_bytes=930119680 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=930574504 ingress_bytes=2339405824 link_bytes=2339405824 token_bytes=1860694184 weight_movement_bytes=1409286144 total_nvlink_bytes=3269980328 weight_to_token_ratio=0.757398 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.309969610e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879441408 weight_movement_bytes=352321536 total_nvlink_bytes=2231762944 weight_to_token_ratio=0.187461 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458768 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982864 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879506960 weight_movement_bytes=352321536 total_nvlink_bytes=2231828496 weight_to_token_ratio=0.187454 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938999808 dispatch_meta_bytes=458752 combine_payload_bytes=938999808 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939458560 ingress_bytes=1995964416 link_bytes=1995964416 token_bytes=1878458368 weight_movement_bytes=1056964608 total_nvlink_bytes=2935422976 weight_to_token_ratio=0.562677 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117656757e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[4, 0, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130916 payload_rows_remote=114534 dup_rows=156 dup_rows_remote=146 dedup_ratio=0.001190 router_tokens_per_expert=[8158, 0, 5, 0, 0, 8173, 8187, 1, 4, 3, 8185, 1, 8186, 6, 4, 8188, 16, 3, 4, 8169, 3, 0, 31, 4, 0, 5, 14, 0, 0, 0, 0, 8186] payload_rows_per_dst=[16326, 16346, 16380, 16382, 16344, 16382, 16384, 16372] dup_rows_per_dst=[0, 2, 6, 10, 40, 68, 30, 0] combine_rows_per_src=[16326, 16346, 16380, 16382, 16344, 16382, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130124 payload_rows_remote=113750 dup_rows=948 dup_rows_remote=910 dedup_ratio=0.007233 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8187] payload_rows_per_dst=[15490, 16384, 16348, 16384, 16376, 16384, 16384, 16374] dup_rows_per_dst=[0, 0, 0, 10, 6, 882, 12, 38] combine_rows_per_src=[15490, 16384, 16348, 16384, 16376, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=920469504 dispatch_meta_bytes=458152 combine_payload_bytes=920469504 prefetch_bytes=176160768 reduce_bytes=1233125376 egress_bytes=920927656 ingress_bytes=2329755648 link_bytes=2329755648 token_bytes=1841397160 weight_movement_bytes=1409286144 total_nvlink_bytes=3250683304 weight_to_token_ratio=0.765335 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.304565914e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114680 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8192, 8191, 0, 8191, 0, 1, 8190, 0, 0, 0, 3, 0, 8192, 1, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 8, 0, 0] combine_rows_per_src=[16384, 16382, 16382, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938262528 dispatch_meta_bytes=458720 combine_payload_bytes=938262528 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=938721248 ingress_bytes=2171387904 link_bytes=2171387904 token_bytes=1876983776 weight_movement_bytes=1233125376 total_nvlink_bytes=3110109152 weight_to_token_ratio=0.656972 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.215886588e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=931840000 dispatch_meta_bytes=458640 combine_payload_bytes=931840000 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=932298640 ingress_bytes=1284161536 link_bytes=1284161536 token_bytes=1864138640 weight_movement_bytes=352321536 total_nvlink_bytes=2216460176 weight_to_token_ratio=0.189000 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.190768566e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130736 payload_rows_remote=114352 dup_rows=336 dup_rows_remote=264 dedup_ratio=0.002563 router_tokens_per_expert=[8165, 0, 10, 0, 0, 8155, 8184, 0, 9, 5, 8178, 3, 8182, 9, 14, 8185, 29, 0, 6, 8167, 6, 1, 4, 7, 0, 8, 22, 0, 0, 5, 0, 8182] payload_rows_per_dst=[16384, 16368, 16360, 16364, 16346, 16194, 16346, 16374] dup_rows_per_dst=[72, 92, 16, 74, 58, 18, 6, 0] combine_rows_per_src=[16384, 16368, 16360, 16364, 16346, 16194, 16346, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131012 payload_rows_remote=114628 dup_rows=60 dup_rows_remote=60 dedup_ratio=0.000458 router_tokens_per_expert=[8191, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 8192, 0, 2, 8192, 0, 0, 0, 8164, 0, 0, 27, 0, 0, 0, 0, 1, 0, 0, 0, 8191] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16328, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 4, 0, 54, 2, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16328, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131060 payload_rows_remote=114676 dup_rows=12 dup_rows_remote=10 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 8191, 0, 0, 8192, 8192, 0, 8183, 0, 6, 8189, 0, 0, 0, 5, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16378, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 10, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16378, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939917312 ingress_bytes=1644101632 link_bytes=1644101632 token_bytes=1879375872 weight_movement_bytes=704643072 total_nvlink_bytes=2584018944 weight_to_token_ratio=0.374935 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206282857e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131052 payload_rows_remote=114668 dup_rows=20 dup_rows_remote=6 dedup_ratio=0.000153 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 8192, 0, 3, 8192, 0, 0, 0, 8182, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16364, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 6, 0, 14, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16364, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131054 payload_rows_remote=114670 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000137 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8192, 0, 0, 8192, 0, 8184, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16368, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 18, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16368, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=936771584 dispatch_meta_bytes=458464 combine_payload_bytes=936771584 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937230048 ingress_bytes=1289093120 link_bytes=1289093120 token_bytes=1874001632 weight_movement_bytes=352321536 total_nvlink_bytes=2226323168 weight_to_token_ratio=0.188005 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.218383378e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=458752 combine_payload_bytes=939032576 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=939491328 ingress_bytes=1819836416 link_bytes=1819836416 token_bytes=1878523904 weight_movement_bytes=880803840 total_nvlink_bytes=2759327744 weight_to_token_ratio=0.468881 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019032429e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458744 combine_payload_bytes=939425792 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939884536 ingress_bytes=1996390400 link_bytes=1996390400 token_bytes=1879310328 weight_movement_bytes=1056964608 total_nvlink_bytes=2936274936 weight_to_token_ratio=0.562422 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117895291e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458696 combine_payload_bytes=939360256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939818952 ingress_bytes=1291681792 link_bytes=1291681792 token_bytes=1879179208 weight_movement_bytes=352321536 total_nvlink_bytes=2231500744 weight_to_token_ratio=0.187487 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232878861e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130630 payload_rows_remote=114272 dup_rows=442 dup_rows_remote=442 dedup_ratio=0.003372 router_tokens_per_expert=[7, 3, 19, 5, 22, 0, 0, 8185, 0, 9, 8185, 8182, 6, 8169, 0, 4, 8187, 0, 0, 0, 3, 0, 8017, 1, 8179, 0, 3, 172, 6, 8164, 5, 3] payload_rows_per_dst=[16384, 16370, 16378, 16358, 16374, 16034, 16384, 16348] dup_rows_per_dst=[54, 44, 4, 0, 0, 8, 324, 8] combine_rows_per_src=[16384, 16370, 16378, 16358, 16374, 16034, 16384, 16348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130768 payload_rows_remote=114388 dup_rows=304 dup_rows_remote=266 dedup_ratio=0.002319 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8, 8192, 0, 11, 8192, 0, 0, 0, 8166, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8190] payload_rows_per_dst=[16384, 16342, 16340, 16222, 16332, 16384, 16384, 16380] dup_rows_per_dst=[0, 0, 16, 22, 210, 18, 0, 38] combine_rows_per_src=[16384, 16342, 16340, 16222, 16332, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=458752 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=939835392 ingress_bytes=1996341248 link_bytes=1996341248 token_bytes=1879212032 weight_movement_bytes=1056964608 total_nvlink_bytes=2936176640 weight_to_token_ratio=0.562451 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117867767e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122164 payload_rows_remote=105780 dup_rows=8908 dup_rows_remote=8908 dedup_ratio=0.067963 router_tokens_per_expert=[0, 0, 5, 0, 0, 0, 0, 0, 2747, 8192, 8, 6718, 0, 0, 0, 0, 8192, 8192, 6906, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 8034, 16384, 15826, 16384, 16384, 16384] dup_rows_per_dst=[10, 0, 550, 0, 8348, 0, 0, 0] combine_rows_per_src=[16384, 16384, 8034, 16384, 15826, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=936116224 dispatch_meta_bytes=458856 combine_payload_bytes=936116224 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936575080 ingress_bytes=1288437760 link_bytes=1288437760 token_bytes=1872691304 weight_movement_bytes=352321536 total_nvlink_bytes=2225012840 weight_to_token_ratio=0.188136 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.214713636e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=937066496 dispatch_meta_bytes=458616 combine_payload_bytes=937066496 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937525112 ingress_bytes=1289388032 link_bytes=1289388032 token_bytes=1874591608 weight_movement_bytes=352321536 total_nvlink_bytes=2226913144 weight_to_token_ratio=0.187946 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.220034762e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130518 payload_rows_remote=114134 dup_rows=554 dup_rows_remote=464 dedup_ratio=0.004227 router_tokens_per_expert=[13, 7, 28, 8, 34, 0, 0, 8185, 0, 9, 8181, 8176, 10, 8165, 0, 0, 8185, 0, 0, 0, 0, 0, 7931, 0, 8166, 0, 4, 256, 20, 8148, 8, 2] payload_rows_per_dst=[16384, 16370, 16362, 16350, 16370, 16324, 16012, 16346] dup_rows_per_dst=[90, 76, 0, 0, 0, 370, 8, 10] combine_rows_per_src=[16384, 16370, 16362, 16350, 16370, 16324, 16012, 16346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=866549760 dispatch_meta_bytes=458752 combine_payload_bytes=866549760 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=867008512 ingress_bytes=1218871296 link_bytes=1218871296 token_bytes=1733558272 weight_movement_bytes=352321536 total_nvlink_bytes=2085879808 weight_to_token_ratio=0.203236 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.825170476e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114686 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 8192, 8191, 0, 8190, 0, 2, 8191, 0, 0, 0, 2, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129210 payload_rows_remote=112826 dup_rows=1862 dup_rows_remote=1862 dedup_ratio=0.014206 router_tokens_per_expert=[0, 0, 4, 0, 0, 0, 0, 0, 1896, 8192, 20, 7194, 0, 0, 0, 0, 8192, 8192, 7263, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 7, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16380, 16384, 14526, 16384, 16384, 16384] dup_rows_per_dst=[8, 0, 1840, 0, 0, 0, 0, 14] combine_rows_per_src=[16384, 16384, 16380, 16384, 14526, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8191, 0, 8192, 0, 0, 8192, 0, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 2, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934985728 dispatch_meta_bytes=458392 combine_payload_bytes=934985728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935444120 ingress_bytes=1287307264 link_bytes=1287307264 token_bytes=1870429848 weight_movement_bytes=352321536 total_nvlink_bytes=2222751384 weight_to_token_ratio=0.188364 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.208383330e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131040 payload_rows_remote=114678 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000244 router_tokens_per_expert=[8191, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 6, 8192, 0, 6, 8192, 0, 0, 0, 8181, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8188] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16362, 16384, 16384, 16376] dup_rows_per_dst=[0, 0, 12, 12, 0, 8, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16362, 16384, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458760 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=939966472 ingress_bytes=2348793856 link_bytes=2348793856 token_bytes=1879474184 weight_movement_bytes=1409286144 total_nvlink_bytes=3288760328 weight_to_token_ratio=0.749830 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315226516e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=924270592 dispatch_meta_bytes=458752 combine_payload_bytes=924270592 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=924729344 ingress_bytes=1276592128 link_bytes=1276592128 token_bytes=1848999936 weight_movement_bytes=352321536 total_nvlink_bytes=2201321472 weight_to_token_ratio=0.190547 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.148383041e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458744 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966456 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474168 weight_movement_bytes=352321536 total_nvlink_bytes=2231795704 weight_to_token_ratio=0.187458 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124670 payload_rows_remote=108296 dup_rows=6402 dup_rows_remote=6396 dedup_ratio=0.048843 router_tokens_per_expert=[4, 0, 1, 5, 5, 0, 96, 4, 4544, 8187, 0, 6726, 3, 0, 0, 5, 8175, 8182, 5007, 5, 4, 8178, 0, 12, 0, 0, 4, 8187, 4, 8, 92, 8098] payload_rows_per_dst=[16384, 16384, 16368, 16374, 10024, 16384, 16382, 16370] dup_rows_per_dst=[10, 176, 6172, 6, 0, 4, 0, 34] combine_rows_per_src=[16384, 16384, 16368, 16374, 10024, 16384, 16382, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130832 payload_rows_remote=114448 dup_rows=240 dup_rows_remote=184 dedup_ratio=0.001831 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8190, 0, 1, 8191, 0, 0, 0, 2, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16240, 16292, 16384, 16382, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 34, 68, 38, 4, 40, 56] combine_rows_per_src=[16240, 16292, 16384, 16382, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131060 payload_rows_remote=114676 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 6, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8186, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16372, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16372, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458840 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939901016 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879343192 weight_movement_bytes=352321536 total_nvlink_bytes=2231664728 weight_to_token_ratio=0.187471 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=887160832 dispatch_meta_bytes=458768 combine_payload_bytes=887160832 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=887619600 ingress_bytes=1591803904 link_bytes=1591803904 token_bytes=1774780432 weight_movement_bytes=704643072 total_nvlink_bytes=2479423504 weight_to_token_ratio=0.397031 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.913437411e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458752 combine_payload_bytes=939425792 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939884544 ingress_bytes=1291747328 link_bytes=1291747328 token_bytes=1879310336 weight_movement_bytes=352321536 total_nvlink_bytes=2231631872 weight_to_token_ratio=0.187474 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233245835e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=937558016 dispatch_meta_bytes=458528 combine_payload_bytes=937558016 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938016544 ingress_bytes=1289879552 link_bytes=1289879552 token_bytes=1875574560 weight_movement_bytes=352321536 total_nvlink_bytes=2227896096 weight_to_token_ratio=0.187847 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.222787069e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125162 payload_rows_remote=108778 dup_rows=5910 dup_rows_remote=5892 dedup_ratio=0.045090 router_tokens_per_expert=[8, 0, 0, 9, 8, 0, 141, 8, 5285, 8184, 1, 5662, 5, 0, 0, 8, 8160, 8175, 5292, 9, 7, 8174, 0, 23, 0, 0, 6, 8183, 2, 10, 144, 8032] payload_rows_per_dst=[16384, 16384, 16370, 16366, 10590, 16378, 16378, 16312] dup_rows_per_dst=[18, 250, 5578, 22, 0, 6, 6, 30] combine_rows_per_src=[16384, 16384, 16370, 16366, 10590, 16378, 16378, 16312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128862 payload_rows_remote=112490 dup_rows=2210 dup_rows_remote=14 dedup_ratio=0.016861 router_tokens_per_expert=[0, 0, 6, 0, 0, 0, 0, 0, 3665, 8192, 0, 5619, 0, 0, 0, 0, 8192, 8192, 7093, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16372, 16384, 14186, 16384, 16384, 16384] dup_rows_per_dst=[12, 0, 2196, 0, 0, 0, 0, 2] combine_rows_per_src=[16384, 16384, 16372, 16384, 14186, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114672 dup_rows=16 dup_rows_remote=0 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 8, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8184, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16368, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 16, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16368, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129076 payload_rows_remote=112692 dup_rows=1996 dup_rows_remote=1996 dedup_ratio=0.015228 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2819, 8192, 2, 6369, 0, 0, 0, 0, 8192, 8192, 7194, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 14388, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 1996, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 14388, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=891109376 dispatch_meta_bytes=458680 combine_payload_bytes=891109376 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=891568056 ingress_bytes=1243430912 link_bytes=1243430912 token_bytes=1782677432 weight_movement_bytes=352321536 total_nvlink_bytes=2134998968 weight_to_token_ratio=0.197636 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.962694074e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=921518080 dispatch_meta_bytes=450016 combine_payload_bytes=921518080 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=921968096 ingress_bytes=1978482688 link_bytes=1978482688 token_bytes=1843486176 weight_movement_bytes=1056964608 total_nvlink_bytes=2900450784 weight_to_token_ratio=0.573351 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=20864 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.107867719e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8190, 0, 8190, 0, 2, 8192, 0, 0, 0, 2, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 0, 0] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=458688 combine_payload_bytes=939393024 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939851712 ingress_bytes=1644036096 link_bytes=1644036096 token_bytes=1879244736 weight_movement_bytes=704643072 total_nvlink_bytes=2583887808 weight_to_token_ratio=0.374961 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205915883e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130048 payload_rows_remote=113672 dup_rows=1024 dup_rows_remote=1014 dedup_ratio=0.007812 router_tokens_per_expert=[5, 0, 4, 0, 316, 5, 3, 8181, 4, 23, 5, 5, 8187, 6, 0, 0, 8172, 0, 167, 8186, 5, 0, 6, 8185, 8061, 0, 0, 1, 4, 7819, 0, 8186] payload_rows_per_dst=[16382, 16376, 16384, 16376, 16376, 16384, 16124, 15646] dup_rows_per_dst=[8, 634, 62, 10, 302, 8, 0, 0] combine_rows_per_src=[16382, 16376, 16384, 16376, 16376, 16384, 16124, 15646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127012 payload_rows_remote=110628 dup_rows=4060 dup_rows_remote=4060 dedup_ratio=0.030975 router_tokens_per_expert=[8108, 6723, 5, 8192, 0, 0, 0, 8173, 0, 0, 8192, 7645, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 2112, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[13456, 16346, 15290, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 4060, 0, 0] combine_rows_per_src=[13456, 16346, 15290, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=923172864 dispatch_meta_bytes=458752 combine_payload_bytes=923172864 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=923631616 ingress_bytes=1627815936 link_bytes=1627815936 token_bytes=1846804480 weight_movement_bytes=704643072 total_nvlink_bytes=2551447552 weight_to_token_ratio=0.381547 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.115089758e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118864 payload_rows_remote=102480 dup_rows=12208 dup_rows_remote=12206 dedup_ratio=0.093140 router_tokens_per_expert=[0, 0, 9, 0, 0, 0, 0, 0, 2301, 8190, 14, 7335, 0, 0, 0, 0, 8192, 8192, 6727, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16306, 15958, 4688, 16384, 16380, 16380, 16384, 16384] dup_rows_per_dst=[14, 0, 20, 0, 12170, 0, 2, 2] combine_rows_per_src=[16306, 15958, 4688, 16384, 16380, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879441408 weight_movement_bytes=352321536 total_nvlink_bytes=2231762944 weight_to_token_ratio=0.187461 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=931201024 dispatch_meta_bytes=458744 combine_payload_bytes=931201024 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=931659768 ingress_bytes=1283522560 link_bytes=1283522560 token_bytes=1862860792 weight_movement_bytes=352321536 total_nvlink_bytes=2215182328 weight_to_token_ratio=0.189129 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.187190567e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=906264576 dispatch_meta_bytes=458752 combine_payload_bytes=906264576 prefetch_bytes=0 reduce_bytes=0 egress_bytes=906723328 ingress_bytes=906264576 link_bytes=906723328 token_bytes=1812987904 weight_movement_bytes=0 total_nvlink_bytes=1812987904 weight_to_token_ratio=0.000000 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.077272152e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129458 payload_rows_remote=113074 dup_rows=1614 dup_rows_remote=1580 dedup_ratio=0.012314 router_tokens_per_expert=[7, 0, 7, 0, 347, 8, 2, 8174, 6, 52, 9, 8, 8183, 7, 0, 4, 8164, 0, 358, 8184, 9, 0, 11, 8182, 8015, 0, 0, 1, 7, 7610, 0, 8181] payload_rows_per_dst=[16384, 16374, 16384, 16378, 16306, 16366, 16032, 15234] dup_rows_per_dst=[34, 1042, 128, 10, 396, 4, 0, 0] combine_rows_per_src=[16384, 16374, 16384, 16378, 16306, 16366, 16032, 15234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=839516160 dispatch_meta_bytes=458744 combine_payload_bytes=839516160 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=839974904 ingress_bytes=1544159232 link_bytes=1544159232 token_bytes=1679491064 weight_movement_bytes=704643072 total_nvlink_bytes=2384134136 weight_to_token_ratio=0.419557 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.646647135e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127710 payload_rows_remote=111344 dup_rows=3362 dup_rows_remote=3362 dedup_ratio=0.025650 router_tokens_per_expert=[7819, 7436, 13, 8192, 0, 0, 0, 8183, 0, 0, 8192, 7683, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 1627, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[15338, 16366, 15366, 16384, 16384, 15104, 16384, 16384] dup_rows_per_dst=[1782, 0, 0, 0, 0, 1580, 0, 0] combine_rows_per_src=[15338, 16366, 15366, 16384, 16384, 15104, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28611282408 prefetch_bytes=5637144576 reduce_bytes=4404019200 weight_movement_bytes=10041163776 egress_bytes=14309295592 ingress_bytes=24343150592 link_bytes=24343609344 total_nvlink_bytes=38652446184 weight_to_token_ratio=0.350951 prefetch_share_of_ingress=0.231570 dedup_ratio=0.048963 prefetch_slots=64 unused_prefetch_slots=192 slot_utilization=0.250000 reduce_slots=25 remote_demand=32 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020241 gemm_inner_pad_upper_bound=19558 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.519628175e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=926302208 dispatch_meta_bytes=458616 combine_payload_bytes=926302208 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=926760824 ingress_bytes=1630945280 link_bytes=1630945280 token_bytes=1853063032 weight_movement_bytes=704643072 total_nvlink_bytes=2557706104 weight_to_token_ratio=0.380259 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.132612778e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966464 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474176 weight_movement_bytes=352321536 total_nvlink_bytes=2231795712 weight_to_token_ratio=0.187458 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129322 payload_rows_remote=114688 dup_rows=1750 dup_rows_remote=1750 dedup_ratio=0.013351 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 2685, 8192, 8, 6369, 0, 0, 0, 0, 8192, 8192, 7317, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 14634, 16384, 16384, 16384] dup_rows_per_dst=[6, 0, 1740, 0, 0, 0, 0, 4] combine_rows_per_src=[16384, 16384, 16384, 16384, 14634, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=912130048 dispatch_meta_bytes=458824 combine_payload_bytes=912130048 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=912588872 ingress_bytes=1264451584 link_bytes=1264451584 token_bytes=1824718920 weight_movement_bytes=352321536 total_nvlink_bytes=2177040456 weight_to_token_ratio=0.193083 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.080401062e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130236 payload_rows_remote=113856 dup_rows=836 dup_rows_remote=836 dedup_ratio=0.006378 router_tokens_per_expert=[8028, 7959, 41, 8145, 8, 11, 10, 8179, 4, 0, 8176, 8067, 6, 8184, 0, 0, 0, 4, 7, 0, 4, 7, 504, 0, 0, 0, 8187, 5, 0, 0, 0, 0] payload_rows_per_dst=[15920, 16370, 16142, 16380, 16366, 16384, 16384, 16290] dup_rows_per_dst=[80, 46, 0, 0, 8, 702, 0, 0] combine_rows_per_src=[15920, 16370, 16142, 16380, 16366, 16384, 16384, 16290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128556 payload_rows_remote=112172 dup_rows=2516 dup_rows_remote=628 dedup_ratio=0.019196 router_tokens_per_expert=[0, 0, 0, 0, 5, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8187, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16374, 14686, 16194, 16378, 15772, 16384, 16384, 16384] dup_rows_per_dst=[0, 10, 0, 0, 0, 6, 612, 1888] combine_rows_per_src=[16374, 14686, 16194, 16378, 15772, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 4, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939982848 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879506944 weight_movement_bytes=704643072 total_nvlink_bytes=2584150016 weight_to_token_ratio=0.374908 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28500337776 prefetch_bytes=5989466112 reduce_bytes=3170893824 weight_movement_bytes=9160359936 egress_bytes=14253827184 ingress_bytes=23406870528 link_bytes=23406870528 total_nvlink_bytes=37660697712 weight_to_token_ratio=0.321412 prefetch_share_of_ingress=0.255885 dedup_ratio=0.048628 prefetch_slots=68 unused_prefetch_slots=188 slot_utilization=0.265625 reduce_slots=18 remote_demand=34 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018939 gemm_inner_pad_upper_bound=21209 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.191794028e-05 prefetch_bytes_per_gemm_flop=2.096156881e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=465752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=939989848 ingress_bytes=2348810240 link_bytes=2348810240 token_bytes=1879513944 weight_movement_bytes=1409286144 total_nvlink_bytes=3288800088 weight_to_token_ratio=0.749814 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=27008 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315235690e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=932708352 dispatch_meta_bytes=458768 combine_payload_bytes=932708352 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933167120 ingress_bytes=1285029888 link_bytes=1285029888 token_bytes=1865875472 weight_movement_bytes=352321536 total_nvlink_bytes=2218197008 weight_to_token_ratio=0.188824 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.195630975e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=918913024 dispatch_meta_bytes=451200 combine_payload_bytes=918913024 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=919364224 ingress_bytes=1623556096 link_bytes=1623556096 token_bytes=1838277248 weight_movement_bytes=704643072 total_nvlink_bytes=2542920320 weight_to_token_ratio=0.383317 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.091236432e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115768 payload_rows_remote=99400 dup_rows=15304 dup_rows_remote=86 dedup_ratio=0.116760 router_tokens_per_expert=[8050, 7685, 58, 8123, 13, 16, 15, 8106, 6, 0, 8163, 8149, 9, 8180, 0, 0, 0, 7, 11, 0, 1, 12, 739, 0, 0, 0, 8184, 8, 1, 0, 0, 0] payload_rows_per_dst=[16368, 16230, 16310, 16378, 16348, 1502, 16384, 16248] dup_rows_per_dst=[15218, 70, 0, 0, 14, 2, 0, 0] combine_rows_per_src=[16368, 16230, 16310, 16378, 16348, 1502, 16384, 16248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128472 payload_rows_remote=112592 dup_rows=2600 dup_rows_remote=2600 dedup_ratio=0.019836 router_tokens_per_expert=[8084, 7205, 6, 8192, 0, 0, 0, 8125, 0, 0, 8192, 7940, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 1407, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[14422, 16250, 15880, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 2600, 0, 0] combine_rows_per_src=[14422, 16250, 15880, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=12 export_slots=12 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 2, 0, 2, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28553804760 prefetch_bytes=4932501504 reduce_bytes=4404019200 weight_movement_bytes=9336520704 egress_bytes=14280571864 ingress_bytes=23609753600 link_bytes=23609753600 total_nvlink_bytes=37890325464 weight_to_token_ratio=0.326980 prefetch_share_of_ingress=0.208918 dedup_ratio=0.048518 prefetch_slots=56 unused_prefetch_slots=200 slot_utilization=0.218750 reduce_slots=25 remote_demand=28 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019650 gemm_inner_pad_upper_bound=20193 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.262797810e-05 prefetch_bytes_per_gemm_flop=1.726246843e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=814284800 dispatch_meta_bytes=397944 combine_payload_bytes=814284800 prefetch_bytes=0 reduce_bytes=2113929216 egress_bytes=814682744 ingress_bytes=2928214016 link_bytes=2928214016 token_bytes=1628967544 weight_movement_bytes=2113929216 total_nvlink_bytes=3742896760 weight_to_token_ratio=1.297711 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.639677619e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=922353664 dispatch_meta_bytes=460768 combine_payload_bytes=922353664 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=922814432 ingress_bytes=1979318272 link_bytes=1979318272 token_bytes=1845168096 weight_movement_bytes=1056964608 total_nvlink_bytes=2902132704 weight_to_token_ratio=0.572828 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.108335612e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128596 payload_rows_remote=112212 dup_rows=2476 dup_rows_remote=0 dedup_ratio=0.018890 router_tokens_per_expert=[8084, 7673, 0, 8192, 0, 0, 0, 8184, 0, 0, 8192, 7481, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[15346, 16368, 14962, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 2476, 0, 0] combine_rows_per_src=[15346, 16368, 14962, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122346 payload_rows_remote=105962 dup_rows=8726 dup_rows_remote=8496 dedup_ratio=0.066574 router_tokens_per_expert=[8024, 7384, 8, 8192, 0, 0, 0, 8016, 0, 0, 8192, 7893, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 1439, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[8506, 16038, 15882, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 658, 5246, 10, 32, 2550, 0, 230] combine_rows_per_src=[8506, 16038, 15882, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27748816144 prefetch_bytes=5813305344 reduce_bytes=6165626880 weight_movement_bytes=11978932224 egress_bytes=13878023440 ingress_bytes=25849724928 link_bytes=25849724928 total_nvlink_bytes=39727748368 weight_to_token_ratio=0.431692 prefetch_share_of_ingress=0.224888 dedup_ratio=0.069412 prefetch_slots=66 unused_prefetch_slots=190 slot_utilization=0.257812 reduce_slots=35 remote_demand=33 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018584 gemm_inner_pad_upper_bound=21463 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.046729337e-05 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[4, 2, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28806677248 prefetch_bytes=4756340736 reduce_bytes=8984199168 weight_movement_bytes=13740539904 egress_bytes=14406992640 ingress_bytes=28140224512 link_bytes=28140224512 total_nvlink_bytes=42547217152 weight_to_token_ratio=0.476991 prefetch_share_of_ingress=0.169023 dedup_ratio=0.046261 prefetch_slots=54 unused_prefetch_slots=202 slot_utilization=0.210938 reduce_slots=51 remote_demand=27 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018348 gemm_inner_pad_upper_bound=21590 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.848344435e-05 prefetch_bytes_per_gemm_flop=1.664595170e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=939982848 ingress_bytes=2701131776 link_bytes=2701131776 token_bytes=1879506944 weight_movement_bytes=1761607680 total_nvlink_bytes=3641114624 weight_to_token_ratio=0.937271 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.512521044e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=919240704 dispatch_meta_bytes=448848 combine_payload_bytes=919240704 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=919689552 ingress_bytes=1271562240 link_bytes=1271562240 token_bytes=1838930256 weight_movement_bytes=352321536 total_nvlink_bytes=2191251792 weight_to_token_ratio=0.191590 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.120217768e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=868040704 dispatch_meta_bytes=457832 combine_payload_bytes=868040704 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=868498536 ingress_bytes=1572683776 link_bytes=1572683776 token_bytes=1736539240 weight_movement_bytes=704643072 total_nvlink_bytes=2441182312 weight_to_token_ratio=0.405774 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.806372675e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28849775368 prefetch_bytes=4756340736 reduce_bytes=3523215360 weight_movement_bytes=8279556096 egress_bytes=14428562184 ingress_bytes=22700769280 link_bytes=22700769280 total_nvlink_bytes=37129331464 weight_to_token_ratio=0.286989 prefetch_share_of_ingress=0.209523 dedup_ratio=0.038697 prefetch_slots=54 unused_prefetch_slots=202 slot_utilization=0.210938 reduce_slots=20 remote_demand=27 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019768 gemm_inner_pad_upper_bound=20320 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.944677013e-05 prefetch_bytes_per_gemm_flop=1.664595170e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27634200024 prefetch_bytes=5460983808 reduce_bytes=4580179968 weight_movement_bytes=10041163776 egress_bytes=13820653016 ingress_bytes=23854710784 link_bytes=23854710784 total_nvlink_bytes=37675363800 weight_to_token_ratio=0.363360 prefetch_share_of_ingress=0.228927 dedup_ratio=0.071396 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=26 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020478 gemm_inner_pad_upper_bound=19177 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.348526434e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128880 payload_rows_remote=112496 dup_rows=2192 dup_rows_remote=2192 dedup_ratio=0.016724 router_tokens_per_expert=[8046, 7311, 7, 8192, 0, 0, 0, 8165, 0, 0, 8192, 7997, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 1240, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[14636, 16330, 15994, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 2192, 0, 0] combine_rows_per_src=[14636, 16330, 15994, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=921567232 dispatch_meta_bytes=458752 combine_payload_bytes=921567232 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=922025984 ingress_bytes=1626210304 link_bytes=1626210304 token_bytes=1843593216 weight_movement_bytes=704643072 total_nvlink_bytes=2548236288 weight_to_token_ratio=0.382212 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.106098889e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28925782136 prefetch_bytes=5284823040 reduce_bytes=7398752256 weight_movement_bytes=12683575296 egress_bytes=14466574456 ingress_bytes=27142782976 link_bytes=27142782976 total_nvlink_bytes=41609357432 weight_to_token_ratio=0.438487 prefetch_share_of_ingress=0.194705 dedup_ratio=0.037704 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=42 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020360 gemm_inner_pad_upper_bound=19177 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.499265919e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none - [2026-09-16 21:37:09] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 795.4 | throughput per GPU (TFLOP/s/GPU): 419.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.107781E+00 | loss scale: 1.0 | grad norm: 5.516 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95978 payload_rows_remote=85746 dup_rows=35094 dup_rows_remote=31684 dedup_ratio=0.267746 router_tokens_per_expert=[4362, 0, 1265, 3661, 2965, 2, 1913, 2998, 73, 3333, 0, 1908, 1641, 2998, 1906, 276, 2, 0, 1478, 0, 0, 1954, 3078, 1880, 0, 1168, 7641, 2425, 2262, 5074, 3276, 5997] payload_rows_per_dst=[13014, 11448, 8976, 10232, 14682, 11144, 15762, 10720] dup_rows_per_dst=[5562, 4308, 1652, 3410, 7886, 2680, 6706, 2890] combine_rows_per_src=[13014, 11448, 8976, 10232, 14682, 11144, 15762, 10720] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97520 payload_rows_remote=82356 dup_rows=33552 dup_rows_remote=27694 dedup_ratio=0.255981 router_tokens_per_expert=[3959, 0, 1287, 2800, 2169, 0, 2332, 2741, 167, 4614, 0, 1354, 1687, 3913, 1753, 317, 4, 20, 1258, 0, 2, 2327, 3359, 2126, 1, 1341, 7105, 1726, 3042, 6078, 2890, 5164] payload_rows_per_dst=[9246, 10862, 10758, 11324, 11482, 13572, 15164, 15112] dup_rows_per_dst=[2050, 3622, 1512, 4016, 1410, 6176, 5858, 8908] combine_rows_per_src=[9246, 10862, 10758, 11324, 11482, 13572, 15164, 15112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94486 payload_rows_remote=79970 dup_rows=36586 dup_rows_remote=29164 dedup_ratio=0.279129 router_tokens_per_expert=[4742, 1, 1665, 3508, 2466, 0, 2741, 2565, 341, 3862, 0, 1652, 1726, 2796, 2217, 337, 2, 6, 1104, 0, 0, 2501, 2607, 1912, 13, 1143, 7507, 1698, 2389, 4900, 3199, 5936] payload_rows_per_dst=[13468, 11422, 14516, 11806, 14354, 10936, 8532, 9452] dup_rows_per_dst=[6364, 4122, 7422, 5552, 6336, 3104, 1962, 1724] combine_rows_per_src=[13468, 11422, 14516, 11806, 14354, 10936, 8532, 9452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=92296 payload_rows_remote=79542 dup_rows=38776 dup_rows_remote=30962 dedup_ratio=0.295837 router_tokens_per_expert=[4352, 5, 1400, 2919, 3230, 8, 2993, 2391, 106, 3363, 0, 1883, 1641, 2322, 2767, 220, 4, 1, 947, 1, 3, 2024, 3307, 1766, 2, 989, 7626, 2177, 3017, 5064, 3247, 5761] payload_rows_per_dst=[12374, 12754, 16092, 12974, 12268, 11776, 5636, 8422] dup_rows_per_dst=[4978, 7814, 9864, 8176, 2322, 3600, 700, 1322] combine_rows_per_src=[12374, 12754, 16092, 12974, 12268, 11776, 5636, 8422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97132 payload_rows_remote=86052 dup_rows=33940 dup_rows_remote=30586 dedup_ratio=0.258942 router_tokens_per_expert=[4685, 0, 1806, 3242, 2530, 0, 2519, 2409, 90, 4024, 0, 1291, 1736, 2751, 2352, 273, 2, 0, 844, 0, 1, 2054, 2916, 1355, 2, 899, 7474, 2404, 3407, 5367, 3462, 5641] payload_rows_per_dst=[12284, 11038, 9538, 10566, 12024, 11080, 15624, 14978] dup_rows_per_dst=[5400, 3878, 1272, 3658, 950, 3354, 5934, 9494] combine_rows_per_src=[12284, 11038, 9538, 10566, 12024, 11080, 15624, 14978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=95722 payload_rows_remote=80790 dup_rows=35350 dup_rows_remote=25394 dedup_ratio=0.269699 router_tokens_per_expert=[4247, 1, 1113, 3373, 1942, 0, 2273, 2140, 108, 3987, 1, 2445, 2185, 3133, 2719, 359, 9, 4, 1451, 0, 1, 1784, 2908, 1775, 1, 1071, 7277, 1750, 3258, 5357, 2802, 6062] payload_rows_per_dst=[8020, 9856, 10636, 11856, 13284, 12666, 14472, 14932] dup_rows_per_dst=[954, 2854, 2446, 4936, 1768, 5148, 7288, 9956] combine_rows_per_src=[8020, 9856, 10636, 11856, 13284, 12666, 14472, 14932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=92640 payload_rows_remote=79074 dup_rows=38432 dup_rows_remote=31766 dedup_ratio=0.293213 router_tokens_per_expert=[4046, 3, 1745, 3829, 2566, 9, 1948, 2475, 132, 2980, 2, 1622, 2231, 1928, 2147, 324, 4, 14, 1512, 1, 4, 1975, 3128, 2825, 1, 998, 7324, 2243, 2674, 4834, 3581, 6431] payload_rows_per_dst=[13566, 13912, 15236, 11984, 13630, 13166, 5798, 5348] dup_rows_per_dst=[6666, 9146, 7898, 5458, 2294, 6284, 686, 0] combine_rows_per_src=[13566, 13912, 15236, 11984, 13630, 13166, 5798, 5348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 2, 4, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=702431232 dispatch_meta_bytes=469720 combine_payload_bytes=702431232 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=702900952 ingress_bytes=1054752768 link_bytes=1054752768 token_bytes=1405332184 weight_movement_bytes=352321536 total_nvlink_bytes=1757653720 weight_to_token_ratio=0.250703 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=12416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.906175225e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=674660352 dispatch_meta_bytes=440200 combine_payload_bytes=674660352 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=675100552 ingress_bytes=1555464192 link_bytes=1555464192 token_bytes=1349760904 weight_movement_bytes=880803840 total_nvlink_bytes=2230564744 weight_to_token_ratio=0.652563 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=19968 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.709950192e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=99372 payload_rows_remote=86340 dup_rows=31700 dup_rows_remote=28350 dedup_ratio=0.241852 router_tokens_per_expert=[3574, 0, 1568, 3421, 2424, 1, 3004, 1589, 46, 4919, 0, 1662, 1696, 3252, 2417, 197, 1, 6, 563, 0, 0, 2522, 3617, 2054, 2, 832, 7791, 1850, 2016, 5351, 3233, 5928] payload_rows_per_dst=[12374, 10466, 11370, 11258, 13032, 12240, 15818, 12814] dup_rows_per_dst=[4752, 3570, 1884, 3866, 3350, 4146, 5132, 5000] combine_rows_per_src=[12374, 10466, 11370, 11258, 13032, 12240, 15818, 12814] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=655114240 dispatch_meta_bytes=436536 combine_payload_bytes=655114240 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=655550776 ingress_bytes=1007435776 link_bytes=1007435776 token_bytes=1310665016 weight_movement_bytes=352321536 total_nvlink_bytes=1662986552 weight_to_token_ratio=0.268811 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=16128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.641219821e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=647774208 dispatch_meta_bytes=443360 combine_payload_bytes=647774208 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=648217568 ingress_bytes=1352417280 link_bytes=1352417280 token_bytes=1295991776 weight_movement_bytes=704643072 total_nvlink_bytes=2000634848 weight_to_token_ratio=0.543710 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=13824 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.572972241e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=651608064 dispatch_meta_bytes=442016 combine_payload_bytes=651608064 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=652050080 ingress_bytes=1003929600 link_bytes=1003929600 token_bytes=1303658144 weight_movement_bytes=352321536 total_nvlink_bytes=1655979680 weight_to_token_ratio=0.270256 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=10240 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.621586699e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=704937984 dispatch_meta_bytes=466552 combine_payload_bytes=704937984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=705404536 ingress_bytes=1057259520 link_bytes=1057259520 token_bytes=1410342520 weight_movement_bytes=352321536 total_nvlink_bytes=1762664056 weight_to_token_ratio=0.249813 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=13184 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.920211990e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=661831680 dispatch_meta_bytes=424736 combine_payload_bytes=661831680 prefetch_bytes=352321536 reduce_bytes=1585446912 egress_bytes=662256416 ingress_bytes=2599600128 link_bytes=2599600128 token_bytes=1324088096 weight_movement_bytes=1937768448 total_nvlink_bytes=3261856544 weight_to_token_ratio=1.463474 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.455667559e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=707297280 dispatch_meta_bytes=458760 combine_payload_bytes=707297280 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=707756040 ingress_bytes=1411940352 link_bytes=1411940352 token_bytes=1415053320 weight_movement_bytes=704643072 total_nvlink_bytes=2119696392 weight_to_token_ratio=0.497962 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.015152 gemm_active_groups=14 gemm_peak_group_rows=23808 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.906276598e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128448 payload_rows_remote=112064 dup_rows=2624 dup_rows_remote=2556 dedup_ratio=0.020020 router_tokens_per_expert=[808, 0, 2, 8, 8191, 0, 0, 0, 8035, 7738, 8192, 8182, 3880, 5, 223, 3866, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 6, 28, 0, 0, 8178, 0, 2] payload_rows_per_dst=[16096, 16382, 15476, 15002, 16384, 16364, 16384, 16360] dup_rows_per_dst=[1610, 0, 0, 946, 0, 0, 68, 0] combine_rows_per_src=[16096, 16382, 15476, 15002, 16384, 16364, 16384, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127944 payload_rows_remote=113038 dup_rows=3128 dup_rows_remote=1792 dedup_ratio=0.023865 router_tokens_per_expert=[884, 0, 0, 8, 8182, 1, 1, 1, 7926, 7688, 8189, 8187, 3689, 16, 184, 4232, 8191, 0, 0, 1, 1, 1, 1, 0, 0, 1, 5, 0, 1, 8145, 0, 1] payload_rows_per_dst=[15864, 16366, 15376, 14906, 16384, 16376, 16380, 16292] dup_rows_per_dst=[1772, 4, 0, 1336, 0, 4, 10, 2] combine_rows_per_src=[15864, 16366, 15376, 14906, 16384, 16376, 16380, 16292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127276 payload_rows_remote=112400 dup_rows=3796 dup_rows_remote=3796 dedup_ratio=0.028961 router_tokens_per_expert=[657, 0, 14, 0, 8026, 0, 0, 0, 7843, 7438, 8176, 8188, 4550, 6, 472, 3767, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 52, 0, 0, 8145, 0, 9] payload_rows_per_dst=[15716, 16052, 14876, 15196, 16384, 16376, 16380, 16296] dup_rows_per_dst=[1312, 0, 0, 2394, 0, 0, 78, 12] combine_rows_per_src=[15716, 16052, 14876, 15196, 16384, 16376, 16380, 16296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124898 payload_rows_remote=109440 dup_rows=6174 dup_rows_remote=5188 dedup_ratio=0.047104 router_tokens_per_expert=[501, 0, 1, 3, 8153, 0, 1, 0, 7717, 7015, 8062, 8188, 4364, 41, 119, 4804, 8188, 0, 0, 1, 0, 0, 1, 5, 1, 1, 230, 0, 0, 8139, 1, 0] payload_rows_per_dst=[15458, 16382, 14646, 13538, 16378, 16378, 15838, 16280] dup_rows_per_dst=[986, 662, 2582, 1910, 0, 10, 24, 0] combine_rows_per_src=[15458, 16382, 14646, 13538, 16378, 16378, 15838, 16280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128022 payload_rows_remote=111706 dup_rows=3050 dup_rows_remote=3048 dedup_ratio=0.023270 router_tokens_per_expert=[342, 0, 1, 6, 8157, 1, 1, 0, 7698, 7530, 8164, 8188, 4253, 78, 136, 4621, 8191, 0, 1, 1, 1, 0, 1, 0, 1, 3, 8, 0, 0, 8152, 1, 0] payload_rows_per_dst=[15410, 16316, 15060, 15832, 16384, 16378, 16336, 16306] dup_rows_per_dst=[684, 2, 0, 2344, 2, 2, 16, 0] combine_rows_per_src=[15410, 16316, 15060, 15832, 16384, 16378, 16336, 16306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127384 payload_rows_remote=111006 dup_rows=3688 dup_rows_remote=3686 dedup_ratio=0.028137 router_tokens_per_expert=[836, 0, 22, 16, 8107, 2, 0, 0, 7952, 7482, 8180, 8187, 4747, 12, 335, 3182, 8190, 2, 0, 1, 1, 0, 2, 0, 0, 7, 181, 1, 0, 8085, 2, 4] payload_rows_per_dst=[15936, 16218, 14964, 14940, 16384, 16378, 16384, 16180] dup_rows_per_dst=[1716, 0, 0, 1612, 2, 2, 354, 2] combine_rows_per_src=[15936, 16218, 14964, 14940, 16384, 16378, 16384, 16180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=926007296 dispatch_meta_bytes=459320 combine_payload_bytes=926007296 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=926466616 ingress_bytes=1806811136 link_bytes=1806811136 token_bytes=1852473912 weight_movement_bytes=880803840 total_nvlink_bytes=2733277752 weight_to_token_ratio=0.475474 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022727 gemm_active_groups=11 gemm_peak_group_rows=17536 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.011738816e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=918028288 dispatch_meta_bytes=458480 combine_payload_bytes=918028288 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=918486768 ingress_bytes=1270349824 link_bytes=1270349824 token_bytes=1836515056 weight_movement_bytes=352321536 total_nvlink_bytes=2188836592 weight_to_token_ratio=0.191842 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.113428744e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=12 export_slots=12 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 1, 4, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121538 payload_rows_remote=105208 dup_rows=9534 dup_rows_remote=8890 dedup_ratio=0.072739 router_tokens_per_expert=[714, 0, 2, 27, 8167, 0, 0, 0, 8120, 7797, 8168, 8177, 5401, 4, 440, 2114, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 8165, 0, 0] payload_rows_per_dst=[9294, 16334, 15888, 14612, 16384, 16354, 16342, 16330] dup_rows_per_dst=[674, 0, 6856, 1196, 6, 68, 90, 644] combine_rows_per_src=[9294, 16334, 15888, 14612, 16384, 16354, 16342, 16330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=920780800 dispatch_meta_bytes=464784 combine_payload_bytes=920780800 prefetch_bytes=176160768 reduce_bytes=2113929216 egress_bytes=921245584 ingress_bytes=3210870784 link_bytes=3210870784 token_bytes=1842026384 weight_movement_bytes=2290089984 total_nvlink_bytes=4132116368 weight_to_token_ratio=1.243245 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.797953611e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=896532480 dispatch_meta_bytes=458512 combine_payload_bytes=896532480 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=896990992 ingress_bytes=1248854016 link_bytes=1248854016 token_bytes=1793523472 weight_movement_bytes=352321536 total_nvlink_bytes=2145845008 weight_to_token_ratio=0.196441 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.993061192e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=915095552 dispatch_meta_bytes=459016 combine_payload_bytes=915095552 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=915554568 ingress_bytes=1267417088 link_bytes=1267417088 token_bytes=1830650120 weight_movement_bytes=352321536 total_nvlink_bytes=2182971656 weight_to_token_ratio=0.192457 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.097006647e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=909361152 dispatch_meta_bytes=458768 combine_payload_bytes=909361152 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=909819920 ingress_bytes=1614004224 link_bytes=1614004224 token_bytes=1819181072 weight_movement_bytes=704643072 total_nvlink_bytes=2523824144 weight_to_token_ratio=0.387341 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.037749936e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=861863936 dispatch_meta_bytes=456392 combine_payload_bytes=861863936 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=862320328 ingress_bytes=1214185472 link_bytes=1214185472 token_bytes=1724184264 weight_movement_bytes=352321536 total_nvlink_bytes=2076505800 weight_to_token_ratio=0.204341 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.798931817e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127468 payload_rows_remote=111088 dup_rows=3604 dup_rows_remote=3604 dedup_ratio=0.027496 router_tokens_per_expert=[1224, 0, 0, 44, 8177, 0, 0, 0, 8004, 7674, 8186, 8191, 3925, 3, 485, 3241, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 1, 0, 8184, 0, 0] payload_rows_per_dst=[16044, 16354, 15348, 14216, 16380, 16382, 16376, 16368] dup_rows_per_dst=[2500, 0, 0, 1092, 0, 0, 12, 0] combine_rows_per_src=[16044, 16354, 15348, 14216, 16380, 16382, 16376, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126000 payload_rows_remote=109616 dup_rows=5072 dup_rows_remote=5072 dedup_ratio=0.038696 router_tokens_per_expert=[2, 3, 8185, 1, 8107, 0, 7581, 1, 141, 8191, 0, 0, 8191, 1, 0, 0, 1, 1, 0, 1, 0, 1, 8191, 8184, 1, 1303, 0, 804, 0, 5898, 1, 746] payload_rows_per_dst=[16376, 15164, 16382, 16384, 16384, 16370, 16292, 12648] dup_rows_per_dst=[6, 0, 282, 0, 4, 0, 4138, 642] combine_rows_per_src=[16376, 15164, 16382, 16384, 16384, 16370, 16292, 12648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118338 payload_rows_remote=102080 dup_rows=12734 dup_rows_remote=7380 dedup_ratio=0.097153 router_tokens_per_expert=[0, 6, 8189, 0, 8050, 0, 7022, 0, 331, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8183, 0, 2514, 0, 242, 0, 5795, 0, 628] payload_rows_per_dst=[16380, 5590, 16384, 16384, 16384, 16366, 16258, 14592] dup_rows_per_dst=[10, 0, 662, 0, 0, 0, 5354, 6708] combine_rows_per_src=[16380, 5590, 16384, 16384, 16384, 16366, 16258, 14592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=910032896 dispatch_meta_bytes=458768 combine_payload_bytes=910032896 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=910491664 ingress_bytes=1262354432 link_bytes=1262354432 token_bytes=1820524560 weight_movement_bytes=352321536 total_nvlink_bytes=2172846096 weight_to_token_ratio=0.193527 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.068657887e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=897974272 dispatch_meta_bytes=458752 combine_payload_bytes=897974272 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=898433024 ingress_bytes=1074135040 link_bytes=1074135040 token_bytes=1796407296 weight_movement_bytes=176160768 total_nvlink_bytes=1972568064 weight_to_token_ratio=0.098063 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.014707858e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126202 payload_rows_remote=109818 dup_rows=4870 dup_rows_remote=4784 dedup_ratio=0.037155 router_tokens_per_expert=[1, 4, 8192, 0, 8006, 0, 7100, 0, 43, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8150, 0, 1172, 0, 872, 0, 6175, 0, 1245] payload_rows_per_dst=[16384, 15072, 16384, 16384, 16384, 16300, 15516, 13778] dup_rows_per_dst=[10, 1096, 86, 0, 0, 0, 2616, 1062] combine_rows_per_src=[16384, 15072, 16384, 16384, 16384, 16300, 15516, 13778] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=836239360 dispatch_meta_bytes=437840 combine_payload_bytes=836239360 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=836677200 ingress_bytes=1188560896 link_bytes=1188560896 token_bytes=1672916560 weight_movement_bytes=352321536 total_nvlink_bytes=2025238096 weight_to_token_ratio=0.210603 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=24704 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.655444888e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116046 payload_rows_remote=99722 dup_rows=15026 dup_rows_remote=1114 dedup_ratio=0.114639 router_tokens_per_expert=[6, 7, 8190, 1, 8110, 1, 7484, 0, 62, 8191, 0, 0, 8191, 1, 0, 1, 1, 1, 0, 1, 0, 1, 8191, 8189, 1, 1068, 0, 1521, 0, 5132, 1, 1184] payload_rows_per_dst=[16382, 16324, 16384, 16384, 16384, 16380, 5820, 11988] dup_rows_per_dst=[26, 13912, 122, 2, 4, 0, 314, 646] combine_rows_per_src=[16382, 16324, 16384, 16384, 16384, 16380, 5820, 11988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112904 payload_rows_remote=96520 dup_rows=18168 dup_rows_remote=18024 dedup_ratio=0.138611 router_tokens_per_expert=[64, 11, 8188, 1, 7887, 0, 7525, 0, 48, 8191, 0, 1, 8188, 1, 0, 0, 1, 1, 0, 1, 0, 0, 8191, 8178, 0, 1174, 0, 764, 0, 6089, 0, 1032] payload_rows_per_dst=[16384, 16382, 14036, 16378, 16384, 16356, 3834, 13150] dup_rows_per_dst=[144, 16836, 50, 0, 4, 0, 42, 1092] combine_rows_per_src=[16384, 16382, 14036, 16378, 16384, 16356, 3834, 13150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125616 payload_rows_remote=109368 dup_rows=5456 dup_rows_remote=5456 dedup_ratio=0.041626 router_tokens_per_expert=[20, 3, 8187, 1, 7942, 0, 7789, 0, 232, 8190, 3, 1, 8189, 1, 2, 0, 0, 2, 0, 2, 0, 0, 8189, 8124, 3, 1590, 1, 845, 0, 5063, 0, 1157] payload_rows_per_dst=[16382, 15578, 16384, 16382, 16382, 16248, 16314, 11946] dup_rows_per_dst=[40, 0, 468, 2, 4, 0, 4448, 494] combine_rows_per_src=[16382, 15578, 16384, 16382, 16382, 16248, 16314, 11946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113544 payload_rows_remote=97290 dup_rows=17528 dup_rows_remote=5186 dedup_ratio=0.133728 router_tokens_per_expert=[18, 3, 8192, 0, 7632, 0, 7424, 0, 157, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8167, 0, 1612, 0, 884, 0, 5741, 0, 1133] payload_rows_per_dst=[16074, 0, 16230, 16384, 16370, 16334, 15898, 16254] dup_rows_per_dst=[42, 0, 432, 4, 0, 350, 4358, 12342] combine_rows_per_src=[16074, 0, 16230, 16384, 16370, 16334, 15898, 16254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=899629056 dispatch_meta_bytes=458408 combine_payload_bytes=899629056 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=900087464 ingress_bytes=1780432896 link_bytes=1780432896 token_bytes=1799716520 weight_movement_bytes=880803840 total_nvlink_bytes=2680520360 weight_to_token_ratio=0.489413 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.969681028e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=816922624 dispatch_meta_bytes=403344 combine_payload_bytes=816922624 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=817325968 ingress_bytes=1873887232 link_bytes=1873887232 token_bytes=1634248592 weight_movement_bytes=1056964608 total_nvlink_bytes=2691213200 weight_to_token_ratio=0.646759 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.049298630e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=790691840 dispatch_meta_bytes=458176 combine_payload_bytes=790691840 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=791150016 ingress_bytes=1495334912 link_bytes=1495334912 token_bytes=1581841856 weight_movement_bytes=704643072 total_nvlink_bytes=2286484928 weight_to_token_ratio=0.445457 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.373251324e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=895942656 dispatch_meta_bytes=459296 combine_payload_bytes=895942656 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=896401952 ingress_bytes=1600585728 link_bytes=1600585728 token_bytes=1792344608 weight_movement_bytes=704643072 total_nvlink_bytes=2496987680 weight_to_token_ratio=0.393140 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.962611959e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=796999680 dispatch_meta_bytes=409904 combine_payload_bytes=796999680 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=797409584 ingress_bytes=1149321216 link_bytes=1149321216 token_bytes=1594409264 weight_movement_bytes=352321536 total_nvlink_bytes=1946730800 weight_to_token_ratio=0.220973 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=24064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.435719059e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119766 payload_rows_remote=103398 dup_rows=11306 dup_rows_remote=11306 dedup_ratio=0.086258 router_tokens_per_expert=[1, 0, 0, 1, 0, 8047, 0, 1468, 3327, 3673, 1, 0, 1, 0, 0, 0, 8183, 2778, 1, 8157, 1, 1, 0, 8156, 13, 36, 8190, 328, 0, 1, 5253, 7919] payload_rows_per_dst=[15840, 16136, 11848, 16368, 16368, 16314, 16384, 10508] dup_rows_per_dst=[2, 2894, 2154, 0, 5504, 2, 750, 0] combine_rows_per_src=[15840, 16136, 11848, 16368, 16368, 16314, 16384, 10508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113478 payload_rows_remote=97094 dup_rows=17594 dup_rows_remote=16598 dedup_ratio=0.134232 router_tokens_per_expert=[0, 0, 0, 0, 0, 8072, 0, 619, 3496, 5251, 0, 0, 0, 0, 0, 0, 8191, 1720, 0, 8120, 3, 0, 0, 8186, 2, 38, 8191, 459, 0, 0, 5535, 7653] payload_rows_per_dst=[15306, 12376, 13672, 16382, 6602, 16372, 16384, 16384] dup_rows_per_dst=[0, 1056, 3822, 0, 788, 6, 996, 10926] combine_rows_per_src=[15306, 12376, 13672, 16382, 6602, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126750 payload_rows_remote=110366 dup_rows=4322 dup_rows_remote=4322 dedup_ratio=0.032974 router_tokens_per_expert=[20, 5, 8191, 0, 8122, 0, 7559, 0, 201, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8184, 0, 1363, 0, 450, 0, 6655, 0, 210] payload_rows_per_dst=[16384, 15118, 16384, 16384, 16384, 16368, 16286, 13442] dup_rows_per_dst=[48, 0, 402, 0, 0, 0, 3584, 288] combine_rows_per_src=[16384, 15118, 16384, 16384, 16384, 16368, 16286, 13442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=847036416 dispatch_meta_bytes=458816 combine_payload_bytes=847036416 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=847495232 ingress_bytes=1551679488 link_bytes=1551679488 token_bytes=1694531648 weight_movement_bytes=704643072 total_nvlink_bytes=2399174720 weight_to_token_ratio=0.415834 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.688757430e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=795394048 dispatch_meta_bytes=454768 combine_payload_bytes=795394048 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=795848816 ingress_bytes=1676197888 link_bytes=1676197888 token_bytes=1591242864 weight_movement_bytes=880803840 total_nvlink_bytes=2472046704 weight_to_token_ratio=0.553532 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.386008493e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121540 payload_rows_remote=107548 dup_rows=9532 dup_rows_remote=5732 dedup_ratio=0.072723 router_tokens_per_expert=[0, 0, 0, 0, 0, 8156, 0, 1050, 4408, 4488, 0, 0, 0, 0, 0, 0, 8192, 1668, 0, 8188, 2, 0, 0, 8180, 0, 11, 8182, 150, 0, 0, 4889, 7972] payload_rows_per_dst=[15944, 16340, 13992, 16384, 16376, 16360, 16366, 9778] dup_rows_per_dst=[0, 2072, 3800, 0, 3336, 4, 320, 0] combine_rows_per_src=[15944, 16340, 13992, 16384, 16376, 16360, 16366, 9778] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119440 payload_rows_remote=103112 dup_rows=11632 dup_rows_remote=6816 dedup_ratio=0.088745 router_tokens_per_expert=[2, 0, 0, 1, 0, 8157, 0, 2415, 3182, 3568, 1, 0, 1, 1, 0, 0, 8181, 2114, 1, 8145, 16, 1, 0, 8174, 26, 29, 8188, 349, 0, 1, 4837, 8146] payload_rows_per_dst=[16296, 16328, 11696, 16364, 16348, 16350, 16382, 9676] dup_rows_per_dst=[2, 4816, 1806, 2, 4172, 32, 802, 0] combine_rows_per_src=[16296, 16328, 11696, 16364, 16348, 16350, 16382, 9676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=904118272 dispatch_meta_bytes=458752 combine_payload_bytes=904118272 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=904577024 ingress_bytes=1256439808 link_bytes=1256439808 token_bytes=1808695296 weight_movement_bytes=352321536 total_nvlink_bytes=2161016832 weight_to_token_ratio=0.194793 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.035538461e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112158 payload_rows_remote=95788 dup_rows=18914 dup_rows_remote=9412 dedup_ratio=0.144302 router_tokens_per_expert=[0, 0, 0, 0, 0, 8123, 0, 949, 5172, 4428, 0, 0, 0, 0, 0, 0, 8192, 1573, 0, 8172, 4, 0, 0, 8138, 3, 19, 8168, 166, 0, 0, 4764, 7665] payload_rows_per_dst=[15330, 2262, 13374, 16384, 15800, 16276, 16362, 16370] dup_rows_per_dst=[0, 176, 3844, 0, 3052, 334, 2006, 9502] combine_rows_per_src=[15330, 2262, 13374, 16384, 15800, 16276, 16362, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115378 payload_rows_remote=99296 dup_rows=15694 dup_rows_remote=15688 dedup_ratio=0.119736 router_tokens_per_expert=[3, 6, 0, 2, 2, 8156, 0, 631, 3633, 3853, 1, 0, 1, 0, 0, 0, 8184, 2740, 2, 8171, 3, 1, 0, 8040, 6, 19, 8190, 505, 0, 2, 5531, 7854] payload_rows_per_dst=[15720, 16338, 12360, 16370, 5934, 16082, 16382, 16192] dup_rows_per_dst=[10, 1240, 2614, 0, 314, 6, 1058, 10452] combine_rows_per_src=[15720, 16338, 12360, 16370, 5934, 16082, 16382, 16192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111234 payload_rows_remote=95030 dup_rows=19838 dup_rows_remote=14788 dedup_ratio=0.151352 router_tokens_per_expert=[1, 0, 0, 1, 1, 8082, 0, 518, 4099, 4556, 1, 0, 0, 0, 0, 0, 8185, 1601, 1, 8144, 54, 1, 0, 8188, 26, 31, 8180, 215, 0, 1, 5649, 8001] payload_rows_per_dst=[16204, 16340, 14116, 16370, 16234, 16378, 9540, 6052] dup_rows_per_dst=[5050, 5982, 4950, 68, 3190, 306, 292, 0] combine_rows_per_src=[16204, 16340, 14116, 16370, 16234, 16378, 9540, 6052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=881033216 dispatch_meta_bytes=453120 combine_payload_bytes=881033216 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=881486336 ingress_bytes=1409515520 link_bytes=1409515520 token_bytes=1762519552 weight_movement_bytes=528482304 total_nvlink_bytes=2291001856 weight_to_token_ratio=0.299845 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.892698551e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=844693504 dispatch_meta_bytes=439712 combine_payload_bytes=844693504 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=845133216 ingress_bytes=1549336576 link_bytes=1549336576 token_bytes=1689826720 weight_movement_bytes=704643072 total_nvlink_bytes=2394469792 weight_to_token_ratio=0.416991 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.675638101e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123672 payload_rows_remote=107288 dup_rows=7400 dup_rows_remote=7398 dedup_ratio=0.056458 router_tokens_per_expert=[1, 1, 1, 1796, 8155, 1816, 8191, 1, 0, 1, 8127, 8190, 1, 0, 1, 8191, 24, 0, 17, 0, 1, 6, 8050, 0, 0, 0, 8190, 0, 1972, 1, 2778, 24] payload_rows_per_dst=[16382, 16338, 16256, 16384, 16382, 16102, 16380, 9448] dup_rows_per_dst=[3596, 3606, 0, 2, 82, 12, 0, 102] combine_rows_per_src=[16382, 16338, 16256, 16384, 16382, 16102, 16380, 9448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=784695296 dispatch_meta_bytes=420800 combine_payload_bytes=784695296 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=785116096 ingress_bytes=1841659904 link_bytes=1841659904 token_bytes=1569811392 weight_movement_bytes=1056964608 total_nvlink_bytes=2626776000 weight_to_token_ratio=0.673307 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=20992 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.031252672e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=813432832 dispatch_meta_bytes=459936 combine_payload_bytes=813432832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=813892768 ingress_bytes=1165754368 link_bytes=1165754368 token_bytes=1627325600 weight_movement_bytes=352321536 total_nvlink_bytes=1979647136 weight_to_token_ratio=0.216503 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.527737851e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111236 payload_rows_remote=94852 dup_rows=19836 dup_rows_remote=19836 dedup_ratio=0.151337 router_tokens_per_expert=[0, 0, 0, 1156, 8190, 1681, 8192, 0, 0, 0, 8167, 8189, 0, 0, 0, 8192, 18, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 8192, 0, 3540, 0, 1937, 4] payload_rows_per_dst=[9844, 3362, 16340, 16384, 16384, 16156, 16384, 16382] dup_rows_per_dst=[980, 0, 7860, 0, 36, 0, 0, 10960] combine_rows_per_src=[9844, 3362, 16340, 16384, 16384, 16156, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=778485760 dispatch_meta_bytes=439272 combine_payload_bytes=778485760 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=778925032 ingress_bytes=1483128832 link_bytes=1483128832 token_bytes=1557410792 weight_movement_bytes=704643072 total_nvlink_bytes=2262053864 weight_to_token_ratio=0.452445 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.304902371e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=878903296 dispatch_meta_bytes=458744 combine_payload_bytes=878903296 prefetch_bytes=352321536 reduce_bytes=176160768 egress_bytes=879362040 ingress_bytes=1407385600 link_bytes=1407385600 token_bytes=1758265336 weight_movement_bytes=528482304 total_nvlink_bytes=2286747640 weight_to_token_ratio=0.300570 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.880771888e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123280 payload_rows_remote=108442 dup_rows=7792 dup_rows_remote=5576 dedup_ratio=0.059448 router_tokens_per_expert=[0, 0, 0, 0, 0, 8049, 0, 309, 4189, 4323, 0, 0, 0, 0, 0, 0, 8190, 1205, 0, 8118, 16, 0, 0, 8109, 7, 14, 8181, 322, 0, 0, 6820, 7684] payload_rows_per_dst=[15368, 16110, 13942, 16380, 14838, 16218, 16380, 14044] dup_rows_per_dst=[0, 606, 3082, 0, 2216, 32, 668, 1188] combine_rows_per_src=[15368, 16110, 13942, 16380, 14838, 16218, 16380, 14044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=777027584 dispatch_meta_bytes=458752 combine_payload_bytes=777027584 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=777486336 ingress_bytes=1305509888 link_bytes=1305509888 token_bytes=1554513920 weight_movement_bytes=528482304 total_nvlink_bytes=2082996224 weight_to_token_ratio=0.339966 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.310310426e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118930 payload_rows_remote=102560 dup_rows=12142 dup_rows_remote=2810 dedup_ratio=0.092636 router_tokens_per_expert=[1, 1, 1, 1327, 8139, 2268, 8191, 1, 0, 3, 8045, 8183, 1, 0, 1, 8191, 0, 0, 26, 0, 1, 2, 8088, 0, 0, 1, 8191, 0, 2417, 1, 2440, 16] payload_rows_per_dst=[16376, 16370, 11210, 16384, 16382, 16180, 16382, 9646] dup_rows_per_dst=[2650, 9332, 0, 2, 52, 2, 2, 102] combine_rows_per_src=[16376, 16370, 11210, 16384, 16382, 16180, 16382, 9646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123028 payload_rows_remote=106800 dup_rows=8044 dup_rows_remote=8040 dedup_ratio=0.061371 router_tokens_per_expert=[0, 0, 0, 1450, 8049, 2601, 8192, 0, 2, 0, 8114, 8192, 0, 1, 0, 8192, 0, 0, 1, 0, 0, 0, 7772, 0, 0, 0, 8192, 0, 2656, 0, 2120, 2] payload_rows_per_dst=[16384, 16362, 16228, 16384, 16384, 15544, 16384, 9358] dup_rows_per_dst=[2900, 4938, 4, 2, 2, 0, 0, 198] combine_rows_per_src=[16384, 16362, 16228, 16384, 16384, 15544, 16384, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 4, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=103974 payload_rows_remote=87590 dup_rows=27098 dup_rows_remote=16170 dedup_ratio=0.206741 router_tokens_per_expert=[0, 0, 0, 1194, 8141, 1840, 8192, 0, 0, 2, 8097, 8186, 0, 1, 0, 8192, 10, 0, 8, 0, 0, 0, 7966, 0, 0, 0, 8192, 0, 3517, 0, 1986, 12] payload_rows_per_dst=[2388, 3680, 16380, 16384, 16384, 15990, 16384, 16384] dup_rows_per_dst=[0, 0, 14012, 2, 36, 2120, 0, 10928] combine_rows_per_src=[2388, 3680, 16380, 16384, 16384, 15990, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=888356864 dispatch_meta_bytes=456072 combine_payload_bytes=888356864 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=888812936 ingress_bytes=2297643008 link_bytes=2297643008 token_bytes=1777169800 weight_movement_bytes=1409286144 total_nvlink_bytes=3186455944 weight_to_token_ratio=0.792995 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.286584176e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119310 payload_rows_remote=103026 dup_rows=11762 dup_rows_remote=11658 dedup_ratio=0.089737 router_tokens_per_expert=[2, 2, 2, 1680, 8054, 2504, 8190, 1, 8, 14, 8152, 8186, 2, 3, 2, 8188, 12, 0, 7, 0, 2, 56, 8136, 0, 0, 1, 8190, 0, 2178, 2, 1955, 7] payload_rows_per_dst=[16380, 5992, 16342, 16384, 16380, 16284, 16380, 15168] dup_rows_per_dst=[3364, 1008, 6, 6, 38, 104, 2, 7234] combine_rows_per_src=[16380, 5992, 16342, 16384, 16380, 16284, 16380, 15168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[2, 2, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108712 payload_rows_remote=92352 dup_rows=22360 dup_rows_remote=19554 dedup_ratio=0.170593 router_tokens_per_expert=[1, 1, 1, 1410, 8131, 1565, 8191, 1, 2, 34, 7966, 8170, 1, 17, 1, 8182, 12, 0, 45, 0, 1, 0, 8176, 0, 0, 9, 8191, 0, 2310, 1, 3024, 93] payload_rows_per_dst=[16360, 16384, 108, 16352, 16036, 16354, 16378, 10740] dup_rows_per_dst=[2806, 19388, 8, 18, 14, 0, 10, 116] combine_rows_per_src=[16360, 16384, 108, 16352, 16036, 16354, 16378, 10740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=840171520 dispatch_meta_bytes=421480 combine_payload_bytes=840171520 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=840593000 ingress_bytes=1897136128 link_bytes=1897136128 token_bytes=1680764520 weight_movement_bytes=1056964608 total_nvlink_bytes=2737729128 weight_to_token_ratio=0.628859 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=21248 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.062317041e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124346 payload_rows_remote=108004 dup_rows=6726 dup_rows_remote=6718 dedup_ratio=0.051315 router_tokens_per_expert=[5134, 27, 8190, 0, 8191, 8188, 7922, 1, 1, 2127, 0, 1, 0, 1, 3, 8171, 1, 623, 1, 1, 8191, 0, 0, 0, 0, 0, 8158, 602, 0, 1, 0, 1] payload_rows_per_dst=[10310, 15846, 16382, 16342, 16380, 16382, 16320, 16384] dup_rows_per_dst=[12, 0, 4256, 8, 1248, 0, 1200, 2] combine_rows_per_src=[10310, 15846, 16382, 16342, 16380, 16382, 16320, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=874905600 dispatch_meta_bytes=459360 combine_payload_bytes=874905600 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=875364960 ingress_bytes=2284191744 link_bytes=2284191744 token_bytes=1750270560 weight_movement_bytes=1409286144 total_nvlink_bytes=3159556704 weight_to_token_ratio=0.805182 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=27264 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.279052030e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=717537280 dispatch_meta_bytes=415040 combine_payload_bytes=717537280 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=717952320 ingress_bytes=1069858816 link_bytes=1069858816 token_bytes=1435489600 weight_movement_bytes=352321536 total_nvlink_bytes=1787811136 weight_to_token_ratio=0.245436 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=14464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.990762789e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123644 payload_rows_remote=107398 dup_rows=7428 dup_rows_remote=6820 dedup_ratio=0.056671 router_tokens_per_expert=[5464, 7, 8192, 0, 8192, 8192, 7273, 0, 0, 1687, 0, 0, 0, 0, 0, 8192, 0, 1718, 0, 0, 8192, 0, 0, 0, 0, 0, 8121, 306, 0, 0, 0, 0] payload_rows_per_dst=[10932, 14546, 16384, 16384, 16384, 16384, 16246, 16384] dup_rows_per_dst=[10, 0, 3374, 0, 3436, 0, 608, 0] combine_rows_per_src=[10932, 14546, 16384, 16384, 16384, 16384, 16246, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=843988992 dispatch_meta_bytes=458736 combine_payload_bytes=843988992 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=844447728 ingress_bytes=1196310528 link_bytes=1196310528 token_bytes=1688436720 weight_movement_bytes=352321536 total_nvlink_bytes=2040758256 weight_to_token_ratio=0.208667 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.698839592e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=756547584 dispatch_meta_bytes=447624 combine_payload_bytes=756547584 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=756995208 ingress_bytes=1108869120 link_bytes=1108869120 token_bytes=1513542792 weight_movement_bytes=352321536 total_nvlink_bytes=1865864328 weight_to_token_ratio=0.232779 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=27264 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.209204207e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=884768768 dispatch_meta_bytes=458888 combine_payload_bytes=884768768 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=885227656 ingress_bytes=1237090304 link_bytes=1237090304 token_bytes=1769996424 weight_movement_bytes=352321536 total_nvlink_bytes=2122317960 weight_to_token_ratio=0.199052 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.927189315e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=879804416 dispatch_meta_bytes=456872 combine_payload_bytes=879804416 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=880261288 ingress_bytes=1760608256 link_bytes=1760608256 token_bytes=1760065704 weight_movement_bytes=880803840 total_nvlink_bytes=2640869544 weight_to_token_ratio=0.500438 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.858671320e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122394 payload_rows_remote=106010 dup_rows=8678 dup_rows_remote=8582 dedup_ratio=0.066208 router_tokens_per_expert=[0, 0, 0, 2140, 8180, 1082, 8192, 0, 0, 6, 8153, 8191, 0, 0, 0, 8192, 43, 0, 5, 0, 0, 0, 8112, 0, 0, 1, 8192, 0, 1381, 0, 3629, 37] payload_rows_per_dst=[16382, 12846, 16312, 16384, 16384, 16224, 16384, 11478] dup_rows_per_dst=[4280, 1466, 6, 0, 96, 0, 2, 2828] combine_rows_per_src=[16382, 12846, 16312, 16384, 16384, 16224, 16384, 11478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112976 payload_rows_remote=99470 dup_rows=18096 dup_rows_remote=17384 dedup_ratio=0.138062 router_tokens_per_expert=[5561, 92, 8189, 1, 8191, 8179, 7049, 1, 1, 2034, 1, 1, 1, 5, 11, 8167, 1, 1003, 0, 1, 8191, 0, 0, 0, 0, 0, 8155, 699, 0, 1, 0, 1] payload_rows_per_dst=[16004, 13506, 6090, 16334, 11954, 16382, 16322, 16384] dup_rows_per_dst=[14730, 712, 242, 34, 990, 0, 1386, 2] combine_rows_per_src=[16004, 13506, 6090, 16334, 11954, 16382, 16322, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120818 payload_rows_remote=104880 dup_rows=10254 dup_rows_remote=6830 dedup_ratio=0.078232 router_tokens_per_expert=[3729, 18, 8192, 0, 8192, 8191, 7628, 0, 0, 1764, 0, 0, 0, 0, 0, 8192, 0, 1867, 0, 0, 8192, 0, 0, 0, 0, 0, 8109, 1462, 0, 0, 0, 0] payload_rows_per_dst=[7786, 15256, 15938, 16384, 16384, 16384, 16302, 16384] dup_rows_per_dst=[258, 0, 3424, 0, 3732, 0, 2840, 0] combine_rows_per_src=[7786, 15256, 15938, 16384, 16384, 16384, 16302, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120356 payload_rows_remote=103972 dup_rows=10716 dup_rows_remote=10716 dedup_ratio=0.081757 router_tokens_per_expert=[4870, 40, 8192, 0, 8192, 8179, 7476, 0, 0, 2176, 0, 0, 0, 1, 1, 8192, 0, 1430, 0, 0, 8192, 0, 0, 0, 0, 0, 8118, 477, 0, 0, 0, 0] payload_rows_per_dst=[9814, 12362, 16384, 16384, 16382, 16384, 16262, 16384] dup_rows_per_dst=[6, 0, 4352, 4, 2836, 0, 3518, 0] combine_rows_per_src=[9814, 12362, 16384, 16384, 16382, 16384, 16262, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=868433920 dispatch_meta_bytes=458368 combine_payload_bytes=868433920 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=868892288 ingress_bytes=1220755456 link_bytes=1220755456 token_bytes=1737326208 weight_movement_bytes=352321536 total_nvlink_bytes=2089647744 weight_to_token_ratio=0.202795 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.835720985e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122594 payload_rows_remote=106210 dup_rows=8478 dup_rows_remote=8476 dedup_ratio=0.064682 router_tokens_per_expert=[4774, 12, 8189, 0, 8190, 8186, 7415, 2, 2, 2326, 0, 2, 1, 1, 4, 8174, 0, 1076, 2, 2, 8190, 0, 2, 1, 2, 0, 8154, 827, 0, 2, 0, 0] payload_rows_per_dst=[9570, 14834, 16382, 16350, 16376, 16384, 16314, 16384] dup_rows_per_dst=[2, 0, 4656, 10, 2156, 2, 1652, 0] combine_rows_per_src=[9570, 14834, 16382, 16350, 16376, 16384, 16314, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=814858240 dispatch_meta_bytes=467416 combine_payload_bytes=814858240 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=815325656 ingress_bytes=2224144384 link_bytes=2224144384 token_bytes=1630183896 weight_movement_bytes=1409286144 total_nvlink_bytes=3039470040 weight_to_token_ratio=0.864495 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=28288 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.245428015e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94562 payload_rows_remote=78180 dup_rows=36510 dup_rows_remote=12606 dedup_ratio=0.278549 router_tokens_per_expert=[4570, 191, 8177, 1, 8191, 8164, 7382, 1, 1, 1554, 0, 1, 11, 15, 53, 8190, 1, 2107, 0, 1, 8191, 0, 1, 0, 0, 1, 8151, 580, 0, 1, 0, 0] payload_rows_per_dst=[16382, 16374, 3110, 16328, 4216, 16384, 5388, 16380] dup_rows_per_dst=[23904, 12254, 2, 110, 2, 2, 236, 0] combine_rows_per_src=[16382, 16374, 3110, 16328, 4216, 16384, 5388, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=859176960 dispatch_meta_bytes=446840 combine_payload_bytes=859176960 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=859623800 ingress_bytes=1211498496 link_bytes=1211498496 token_bytes=1718800760 weight_movement_bytes=352321536 total_nvlink_bytes=2071122296 weight_to_token_ratio=0.204981 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=25728 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.783885873e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130992 payload_rows_remote=114610 dup_rows=80 dup_rows_remote=22 dedup_ratio=0.000610 router_tokens_per_expert=[1, 0, 0, 8191, 8182, 1, 0, 0, 2, 1, 8178, 1, 28, 1, 1, 8190, 8178, 0, 8190, 0, 1, 1, 8190, 6, 2, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 16366, 16358, 16382, 16354, 16382, 16384, 16382] dup_rows_per_dst=[0, 0, 6, 58, 0, 14, 2, 0] combine_rows_per_src=[16384, 16366, 16358, 16382, 16354, 16382, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=851738624 dispatch_meta_bytes=458752 combine_payload_bytes=851738624 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=852197376 ingress_bytes=1204060160 link_bytes=1204060160 token_bytes=1703936000 weight_movement_bytes=352321536 total_nvlink_bytes=2056257536 weight_to_token_ratio=0.206769 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.742234297e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131030 payload_rows_remote=114646 dup_rows=42 dup_rows_remote=42 dedup_ratio=0.000320 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8171, 0, 20, 1, 0, 8192, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16342, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 42, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16342, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 2, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=870072320 dispatch_meta_bytes=458744 combine_payload_bytes=870072320 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=870531064 ingress_bytes=1046233088 link_bytes=1046233088 token_bytes=1740603384 weight_movement_bytes=176160768 total_nvlink_bytes=1916764152 weight_to_token_ratio=0.101207 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.858468574e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=640450560 dispatch_meta_bytes=363144 combine_payload_bytes=640450560 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=640813704 ingress_bytes=1345093632 link_bytes=1345093632 token_bytes=1281264264 weight_movement_bytes=704643072 total_nvlink_bytes=1985907336 weight_to_token_ratio=0.549959 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=20736 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.531962869e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938885120 dispatch_meta_bytes=458528 combine_payload_bytes=938885120 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=939343648 ingress_bytes=2348171264 link_bytes=2348171264 token_bytes=1878228768 weight_movement_bytes=1409286144 total_nvlink_bytes=3287514912 weight_to_token_ratio=0.750327 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.314877890e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939180032 dispatch_meta_bytes=458752 combine_payload_bytes=939180032 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=939638784 ingress_bytes=1643823104 link_bytes=1643823104 token_bytes=1878818816 weight_movement_bytes=704643072 total_nvlink_bytes=2583461888 weight_to_token_ratio=0.375046 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204723217e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131022 payload_rows_remote=114644 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000381 router_tokens_per_expert=[1, 0, 0, 8191, 8188, 1, 0, 0, 2, 2, 8179, 1, 19, 0, 1, 8191, 8182, 0, 8190, 0, 1, 1, 8191, 2, 1, 0, 0, 0, 1, 8191, 0, 0] payload_rows_per_dst=[16384, 16378, 16364, 16384, 16362, 16384, 16382, 16384] dup_rows_per_dst=[0, 0, 4, 38, 0, 6, 2, 0] combine_rows_per_src=[16384, 16378, 16364, 16384, 16362, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124894 payload_rows_remote=108524 dup_rows=6178 dup_rows_remote=3952 dedup_ratio=0.047134 router_tokens_per_expert=[5581, 94, 8191, 1, 8192, 8173, 7691, 0, 0, 1857, 0, 0, 0, 1, 8, 8191, 0, 1125, 0, 0, 8192, 0, 0, 0, 0, 0, 8141, 98, 0, 0, 0, 0] payload_rows_per_dst=[11328, 15382, 16382, 16382, 16370, 16384, 16282, 16384] dup_rows_per_dst=[24, 0, 3714, 18, 2226, 0, 196, 0] combine_rows_per_src=[11328, 15382, 16382, 16382, 16370, 16384, 16282, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131046 payload_rows_remote=114688 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000198 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8179, 0, 11, 2, 0, 8192, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16358, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16358, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130924 payload_rows_remote=114542 dup_rows=148 dup_rows_remote=108 dedup_ratio=0.001129 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8166, 0, 25, 2, 0, 8192, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16364, 16384, 16332, 16330, 16380, 16368, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 54, 48, 0, 6, 40] combine_rows_per_src=[16364, 16384, 16332, 16330, 16380, 16368, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=458776 combine_payload_bytes=939163648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939622424 ingress_bytes=1291485184 link_bytes=1291485184 token_bytes=1878786072 weight_movement_bytes=352321536 total_nvlink_bytes=2231107608 weight_to_token_ratio=0.187526 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231777938e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=889028608 dispatch_meta_bytes=449904 combine_payload_bytes=889028608 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=889478512 ingress_bytes=1241350144 link_bytes=1241350144 token_bytes=1778507120 weight_movement_bytes=352321536 total_nvlink_bytes=2130828656 weight_to_token_ratio=0.198100 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=28288 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.951042641e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130972 payload_rows_remote=114590 dup_rows=100 dup_rows_remote=80 dedup_ratio=0.000763 router_tokens_per_expert=[2, 7, 0, 8190, 8181, 2, 0, 3, 0, 1, 8160, 2, 28, 1, 2, 8189, 8190, 0, 8185, 1, 2, 2, 8189, 8, 2, 0, 0, 0, 0, 8188, 1, 0] payload_rows_per_dst=[16384, 16368, 16326, 16382, 16370, 16382, 16382, 16378] dup_rows_per_dst=[14, 4, 0, 58, 2, 20, 2, 0] combine_rows_per_src=[16384, 16368, 16326, 16382, 16370, 16382, 16382, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938328064 dispatch_meta_bytes=458600 combine_payload_bytes=938328064 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938786664 ingress_bytes=1290649600 link_bytes=1290649600 token_bytes=1877114728 weight_movement_bytes=352321536 total_nvlink_bytes=2229436264 weight_to_token_ratio=0.187693 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227099016e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458856 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939982952 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879507048 weight_movement_bytes=352321536 total_nvlink_bytes=2231828584 weight_to_token_ratio=0.187454 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130740 payload_rows_remote=114356 dup_rows=332 dup_rows_remote=326 dedup_ratio=0.002533 router_tokens_per_expert=[1, 0, 0, 8191, 8173, 1, 0, 0, 0, 0, 8146, 1, 80, 0, 1, 8191, 8191, 0, 8188, 10, 1, 1, 8180, 1, 1, 0, 0, 1, 1, 8176, 0, 0] payload_rows_per_dst=[16384, 16372, 16364, 16142, 16382, 16362, 16380, 16354] dup_rows_per_dst=[6, 48, 254, 0, 20, 4, 0, 0] combine_rows_per_src=[16384, 16372, 16364, 16142, 16382, 16362, 16380, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130992 payload_rows_remote=114610 dup_rows=80 dup_rows_remote=18 dedup_ratio=0.000610 router_tokens_per_expert=[1, 1, 8182, 8189, 4, 1, 0, 1, 8190, 8188, 1, 0, 0, 0, 29, 2, 8191, 0, 1, 8191, 8172, 1, 0, 1, 8182, 0, 6, 1, 0, 0, 0, 1] payload_rows_per_dst=[16366, 16382, 16378, 16382, 16384, 16348, 16368, 16384] dup_rows_per_dst=[2, 6, 0, 62, 0, 0, 10, 0] combine_rows_per_src=[16366, 16382, 16378, 16382, 16384, 16348, 16368, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114672 dup_rows=16 dup_rows_remote=4 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 8184, 8190, 0, 2, 0, 0, 8192, 8192, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 8191, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[16368, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 4, 0, 0, 12, 0] combine_rows_per_src=[16368, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938721280 dispatch_meta_bytes=458680 combine_payload_bytes=938721280 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939179960 ingress_bytes=1291042816 link_bytes=1291042816 token_bytes=1877901240 weight_movement_bytes=352321536 total_nvlink_bytes=2230222776 weight_to_token_ratio=0.187615 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229300862e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=936804352 dispatch_meta_bytes=458728 combine_payload_bytes=936804352 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=937263080 ingress_bytes=1289125888 link_bytes=1289125888 token_bytes=1874067432 weight_movement_bytes=352321536 total_nvlink_bytes=2226388968 weight_to_token_ratio=0.187998 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.218566865e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938885120 dispatch_meta_bytes=458512 combine_payload_bytes=938885120 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939343632 ingress_bytes=1291206656 link_bytes=1291206656 token_bytes=1878228752 weight_movement_bytes=352321536 total_nvlink_bytes=2230550288 weight_to_token_ratio=0.187582 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230218297e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=458704 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939851728 ingress_bytes=1467875328 link_bytes=1467875328 token_bytes=1879244752 weight_movement_bytes=528482304 total_nvlink_bytes=2407727056 weight_to_token_ratio=0.281221 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219489115e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130970 payload_rows_remote=114586 dup_rows=102 dup_rows_remote=82 dedup_ratio=0.000778 router_tokens_per_expert=[1, 1, 8177, 8185, 6, 11, 0, 1, 8190, 8184, 1, 0, 1, 0, 27, 0, 8191, 0, 1, 8191, 8168, 0, 0, 1, 8181, 0, 16, 0, 0, 0, 1, 1] payload_rows_per_dst=[16356, 16384, 16370, 16384, 16384, 16338, 16370, 16384] dup_rows_per_dst=[2, 20, 0, 54, 0, 0, 24, 2] combine_rows_per_src=[16356, 16384, 16370, 16384, 16384, 16338, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131046 payload_rows_remote=114662 dup_rows=26 dup_rows_remote=24 dedup_ratio=0.000198 router_tokens_per_expert=[0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8182, 0, 12, 0, 0, 8192, 8190, 0, 8192, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16364, 16384, 16384, 16382, 16380, 16384] dup_rows_per_dst=[0, 0, 0, 24, 2, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16364, 16384, 16384, 16382, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130690 payload_rows_remote=114306 dup_rows=382 dup_rows_remote=382 dedup_ratio=0.002914 router_tokens_per_expert=[0, 0, 8175, 8187, 0, 7, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 8191, 0, 16, 0, 0, 0, 0, 0] payload_rows_per_dst=[16144, 16280, 16350, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 8, 104, 0, 0, 226, 44, 0] combine_rows_per_src=[16144, 16280, 16350, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114674 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 8188, 8189, 0, 5, 0, 0, 8192, 8191, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 8191, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16384, 16382, 16378, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 10, 0, 2, 0, 0, 4, 0] combine_rows_per_src=[16376, 16384, 16382, 16378, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938688512 dispatch_meta_bytes=458672 combine_payload_bytes=938688512 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939147184 ingress_bytes=1291010048 link_bytes=1291010048 token_bytes=1877835696 weight_movement_bytes=352321536 total_nvlink_bytes=2230157232 weight_to_token_ratio=0.187621 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229117375e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=458744 combine_payload_bytes=939311104 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=939769848 ingress_bytes=1996275712 link_bytes=1996275712 token_bytes=1879080952 weight_movement_bytes=1056964608 total_nvlink_bytes=2936045560 weight_to_token_ratio=0.562490 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117831070e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130910 payload_rows_remote=114650 dup_rows=162 dup_rows_remote=162 dedup_ratio=0.001236 router_tokens_per_expert=[2, 2, 8184, 8187, 4, 8, 0, 2, 8187, 8189, 2, 1, 0, 0, 70, 1, 8190, 0, 2, 8189, 8121, 7, 0, 2, 8177, 0, 4, 2, 0, 0, 1, 2] payload_rows_per_dst=[16372, 16384, 16382, 16380, 16382, 16260, 16366, 16384] dup_rows_per_dst=[4, 12, 2, 142, 0, 0, 0, 2] combine_rows_per_src=[16372, 16384, 16382, 16380, 16382, 16260, 16366, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=936394752 dispatch_meta_bytes=458752 combine_payload_bytes=936394752 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936853504 ingress_bytes=1288716288 link_bytes=1288716288 token_bytes=1873248256 weight_movement_bytes=352321536 total_nvlink_bytes=2225569792 weight_to_token_ratio=0.188081 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.216273276e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=458760 combine_payload_bytes=939409408 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=939868168 ingress_bytes=2172534784 link_bytes=2172534784 token_bytes=1879277576 weight_movement_bytes=1233125376 total_nvlink_bytes=3112402952 weight_to_token_ratio=0.656170 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216528793e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116116 payload_rows_remote=111848 dup_rows=14956 dup_rows_remote=14956 dedup_ratio=0.114105 router_tokens_per_expert=[0, 187, 13, 0, 6977, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 1158, 0, 0, 0, 8040, 8192, 0, 0, 0, 2134, 0, 0, 0, 0, 6624, 0, 7635] payload_rows_per_dst=[16384, 13954, 16384, 16384, 16080, 16384, 4268, 16278] dup_rows_per_dst=[400, 0, 0, 2316, 0, 0, 0, 12240] combine_rows_per_src=[16384, 13954, 16384, 16384, 16080, 16384, 4268, 16278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125206 payload_rows_remote=108822 dup_rows=5866 dup_rows_remote=4610 dedup_ratio=0.044754 router_tokens_per_expert=[0, 99, 19, 0, 6875, 0, 1, 1, 8191, 8189, 8191, 1, 1, 1, 0, 627, 0, 1, 2, 8055, 8191, 1, 1, 0, 2504, 7, 0, 0, 0, 6849, 0, 7729] payload_rows_per_dst=[16382, 13752, 16380, 16384, 16112, 16384, 16114, 13698] dup_rows_per_dst=[236, 2, 0, 1256, 4, 2, 4366, 0] combine_rows_per_src=[16382, 13752, 16380, 16384, 16112, 16384, 16114, 13698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130614 payload_rows_remote=114238 dup_rows=458 dup_rows_remote=112 dedup_ratio=0.003494 router_tokens_per_expert=[1, 1, 8164, 8170, 1, 35, 0, 1, 8190, 8186, 1, 0, 0, 0, 7, 6, 8191, 0, 1, 8191, 8181, 0, 0, 1, 8152, 0, 46, 1, 0, 0, 8, 1] payload_rows_per_dst=[16376, 16360, 16298, 16140, 16380, 16364, 16318, 16378] dup_rows_per_dst=[346, 52, 38, 2, 6, 0, 14, 0] combine_rows_per_src=[16376, 16360, 16298, 16140, 16380, 16364, 16318, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=459248 combine_payload_bytes=939212800 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939672048 ingress_bytes=1291534336 link_bytes=1291534336 token_bytes=1878884848 weight_movement_bytes=352321536 total_nvlink_bytes=2231206384 weight_to_token_ratio=0.187516 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232053169e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=916258816 dispatch_meta_bytes=507216 combine_payload_bytes=916258816 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=916766032 ingress_bytes=1268580352 link_bytes=1268580352 token_bytes=1833024848 weight_movement_bytes=352321536 total_nvlink_bytes=2185346384 weight_to_token_ratio=0.192208 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.103520440e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=891469824 dispatch_meta_bytes=453728 combine_payload_bytes=891469824 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=891923552 ingress_bytes=1243791360 link_bytes=1243791360 token_bytes=1783393376 weight_movement_bytes=352321536 total_nvlink_bytes=2135714912 weight_to_token_ratio=0.197557 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.964712432e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=935837696 dispatch_meta_bytes=457400 combine_payload_bytes=935837696 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=936295096 ingress_bytes=1816641536 link_bytes=1816641536 token_bytes=1872132792 weight_movement_bytes=880803840 total_nvlink_bytes=2752936632 weight_to_token_ratio=0.470481 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.017243430e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118550 payload_rows_remote=103262 dup_rows=12522 dup_rows_remote=10900 dedup_ratio=0.095535 router_tokens_per_expert=[0, 287, 22, 0, 7397, 0, 1, 1, 8191, 8190, 8191, 1, 1, 1, 0, 1475, 0, 1, 4, 8115, 8191, 1, 1, 0, 3645, 5, 0, 0, 0, 3952, 0, 7863] payload_rows_per_dst=[16382, 15288, 16384, 13808, 16234, 16384, 16166, 7904] dup_rows_per_dst=[618, 1622, 1230, 2184, 6, 2, 6860, 0] combine_rows_per_src=[16382, 15288, 16384, 13808, 16234, 16384, 16166, 7904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111832 payload_rows_remote=95460 dup_rows=19240 dup_rows_remote=4476 dedup_ratio=0.146790 router_tokens_per_expert=[0, 77, 15, 0, 6869, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 1107, 0, 0, 0, 8033, 8192, 0, 0, 0, 2193, 0, 0, 0, 0, 6505, 0, 7969] payload_rows_per_dst=[13886, 13738, 14614, 16384, 16068, 16384, 4386, 16372] dup_rows_per_dst=[176, 0, 0, 2214, 2070, 16, 0, 14764] combine_rows_per_src=[13886, 13738, 14614, 16384, 16068, 16384, 4386, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131040 payload_rows_remote=114656 dup_rows=32 dup_rows_remote=32 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 8180, 8188, 0, 11, 0, 0, 8192, 8192, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 8189, 0, 6, 0, 0, 0, 1, 0] payload_rows_per_dst=[16360, 16384, 16384, 16376, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 22, 0, 2, 0, 0, 6, 2] combine_rows_per_src=[16360, 16384, 16384, 16376, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119606 payload_rows_remote=103222 dup_rows=11466 dup_rows_remote=11466 dedup_ratio=0.087479 router_tokens_per_expert=[0, 116, 25, 0, 7133, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 2073, 0, 0, 0, 8105, 8192, 0, 0, 0, 4101, 0, 0, 0, 0, 3795, 0, 7420] payload_rows_per_dst=[16384, 14266, 16384, 16384, 16210, 16384, 16004, 7590] dup_rows_per_dst=[282, 0, 0, 4146, 0, 0, 7038, 0] combine_rows_per_src=[16384, 14266, 16384, 16384, 16210, 16384, 16004, 7590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=11 export_slots=11 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 2, 0, 2, 2, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=845922304 dispatch_meta_bytes=456648 combine_payload_bytes=845922304 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=846378952 ingress_bytes=1198243840 link_bytes=1198243840 token_bytes=1692301256 weight_movement_bytes=352321536 total_nvlink_bytes=2044622792 weight_to_token_ratio=0.208191 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.709665332e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=782008320 dispatch_meta_bytes=399744 combine_payload_bytes=782008320 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=782408064 ingress_bytes=1486651392 link_bytes=1486651392 token_bytes=1564416384 weight_movement_bytes=704643072 total_nvlink_bytes=2269059456 weight_to_token_ratio=0.450419 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.324627236e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119536 payload_rows_remote=103152 dup_rows=11536 dup_rows_remote=11532 dedup_ratio=0.088013 router_tokens_per_expert=[0, 177, 23, 1, 7123, 0, 2, 2, 8190, 8189, 8189, 2, 2, 2, 0, 1418, 1, 3, 0, 8132, 8190, 2, 2, 0, 3243, 15, 0, 0, 0, 4919, 0, 7709] payload_rows_per_dst=[16380, 14250, 16382, 16382, 16268, 16384, 11052, 12438] dup_rows_per_dst=[402, 4, 0, 2840, 4, 4, 3520, 4762] combine_rows_per_src=[16380, 14250, 16382, 16382, 16268, 16384, 11052, 12438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=458752 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=939720704 ingress_bytes=1820065792 link_bytes=1820065792 token_bytes=1878982656 weight_movement_bytes=880803840 total_nvlink_bytes=2759786496 weight_to_token_ratio=0.468766 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019160870e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=845594624 dispatch_meta_bytes=458752 combine_payload_bytes=845594624 prefetch_bytes=176160768 reduce_bytes=1937768448 egress_bytes=846053376 ingress_bytes=2959523840 link_bytes=2959523840 token_bytes=1691648000 weight_movement_bytes=2113929216 total_nvlink_bytes=3805577216 weight_to_token_ratio=1.249627 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=28416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.657209814e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115966 payload_rows_remote=99588 dup_rows=15106 dup_rows_remote=15022 dedup_ratio=0.115250 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 7215, 0, 0, 0, 0, 1640, 8174, 8192, 0, 7522, 0, 5, 8192, 0, 0, 0, 0, 43, 0, 8192, 8167, 8192] payload_rows_per_dst=[16384, 16384, 14430, 3280, 16384, 16384, 16378, 16342] dup_rows_per_dst=[0, 2, 2, 0, 15008, 10, 84, 0] combine_rows_per_src=[16384, 16384, 14430, 3280, 16384, 16384, 16378, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124456 payload_rows_remote=108084 dup_rows=6616 dup_rows_remote=396 dedup_ratio=0.050476 router_tokens_per_expert=[2, 0, 2, 1, 2, 0, 0, 1, 0, 72, 5816, 1, 1, 1, 1, 3179, 8114, 8191, 1, 7281, 8, 3, 8190, 0, 0, 0, 5, 177, 1, 8138, 8158, 8190] payload_rows_per_dst=[16384, 16384, 11776, 16372, 14564, 16382, 16382, 16212] dup_rows_per_dst=[6, 4, 2, 6220, 0, 20, 364, 0] combine_rows_per_src=[16384, 16384, 11776, 16372, 14564, 16382, 16382, 16212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=111200 payload_rows_remote=94818 dup_rows=19872 dup_rows_remote=19632 dedup_ratio=0.151611 router_tokens_per_expert=[0, 44, 76, 0, 7075, 0, 1, 1, 8191, 8190, 8191, 1, 1, 1, 0, 850, 0, 1, 12, 8055, 8180, 1, 1, 0, 3421, 2, 0, 0, 0, 5599, 0, 7642] payload_rows_per_dst=[16382, 16220, 16384, 2436, 16126, 16360, 16094, 11198] dup_rows_per_dst=[240, 12968, 546, 70, 10, 2, 6036, 0] combine_rows_per_src=[16382, 16220, 16384, 2436, 16126, 16360, 16094, 11198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=845021184 dispatch_meta_bytes=458736 combine_payload_bytes=845021184 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=845479920 ingress_bytes=1197342720 link_bytes=1197342720 token_bytes=1690501104 weight_movement_bytes=352321536 total_nvlink_bytes=2042822640 weight_to_token_ratio=0.208412 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.704619437e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=815824896 dispatch_meta_bytes=458440 combine_payload_bytes=815824896 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=816283336 ingress_bytes=1168146432 link_bytes=1168146432 token_bytes=1632108232 weight_movement_bytes=352321536 total_nvlink_bytes=1984429768 weight_to_token_ratio=0.215869 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.541132411e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=885424128 dispatch_meta_bytes=433920 combine_payload_bytes=885424128 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=885858048 ingress_bytes=1237745664 link_bytes=1237745664 token_bytes=1771282176 weight_movement_bytes=352321536 total_nvlink_bytes=2123603712 weight_to_token_ratio=0.198908 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=24064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.930859058e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=776749056 dispatch_meta_bytes=457800 combine_payload_bytes=776749056 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=777206856 ingress_bytes=1129070592 link_bytes=1129070592 token_bytes=1553955912 weight_movement_bytes=352321536 total_nvlink_bytes=1906277448 weight_to_token_ratio=0.226726 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.024621 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.322324018e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114832 payload_rows_remote=98448 dup_rows=16240 dup_rows_remote=16236 dedup_ratio=0.123901 router_tokens_per_expert=[3, 0, 2, 1, 2, 0, 0, 1, 0, 76, 5331, 1, 1, 1, 1, 3635, 8098, 8191, 1, 7299, 7, 8, 8190, 0, 0, 0, 0, 160, 1, 8163, 8172, 8191] payload_rows_per_dst=[16384, 16384, 16264, 16374, 372, 16382, 16378, 16294] dup_rows_per_dst=[10, 4, 8780, 7098, 0, 28, 320, 0] combine_rows_per_src=[16384, 16384, 16264, 16374, 372, 16382, 16378, 16294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112670 payload_rows_remote=96286 dup_rows=18402 dup_rows_remote=16372 dedup_ratio=0.140396 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6718, 0, 0, 0, 0, 1535, 8188, 8192, 0, 8111, 0, 0, 8192, 0, 0, 0, 0, 30, 0, 8192, 8186, 8192] payload_rows_per_dst=[16322, 16304, 13436, 3070, 16384, 16234, 14536, 16384] dup_rows_per_dst=[0, 0, 0, 0, 16318, 0, 54, 2030] combine_rows_per_src=[16322, 16304, 13436, 3070, 16384, 16234, 14536, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129028 payload_rows_remote=114106 dup_rows=2044 dup_rows_remote=1900 dedup_ratio=0.015594 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7434, 0, 0, 0, 0, 904, 8188, 8191, 0, 8022, 0, 1, 8192, 0, 0, 0, 0, 48, 0, 8192, 8171, 8192] payload_rows_per_dst=[16384, 16382, 14922, 16378, 15848, 16384, 16376, 16354] dup_rows_per_dst=[0, 0, 144, 1806, 0, 2, 92, 0] combine_rows_per_src=[16384, 16382, 14922, 16378, 15848, 16384, 16376, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126464 payload_rows_remote=110584 dup_rows=4608 dup_rows_remote=4608 dedup_ratio=0.035156 router_tokens_per_expert=[0, 67, 10, 0, 7504, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 750, 0, 0, 0, 7940, 8191, 0, 0, 0, 1533, 0, 0, 0, 0, 6914, 0, 8051] payload_rows_per_dst=[16384, 15008, 16384, 16384, 15880, 16382, 16214, 13828] dup_rows_per_dst=[154, 0, 0, 1500, 0, 0, 2954, 0] combine_rows_per_src=[16384, 15008, 16384, 16384, 15880, 16382, 16214, 13828] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=806486016 dispatch_meta_bytes=458736 combine_payload_bytes=806486016 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=806944752 ingress_bytes=1158807552 link_bytes=1158807552 token_bytes=1613430768 weight_movement_bytes=352321536 total_nvlink_bytes=1965752304 weight_to_token_ratio=0.218368 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.488838581e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=788774912 dispatch_meta_bytes=450632 combine_payload_bytes=788774912 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=789225544 ingress_bytes=2198061056 link_bytes=2198061056 token_bytes=1578000456 weight_movement_bytes=1409286144 total_nvlink_bytes=2987286600 weight_to_token_ratio=0.893083 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.230822440e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120022 payload_rows_remote=103642 dup_rows=11050 dup_rows_remote=11020 dedup_ratio=0.084305 router_tokens_per_expert=[0, 0, 8, 2, 35, 0, 1, 2, 0, 143, 4923, 2, 2, 2, 2, 3990, 8125, 8190, 2, 7144, 13, 4, 8187, 1, 2, 0, 0, 276, 2, 8105, 8183, 8190] payload_rows_per_dst=[16384, 16384, 10136, 13318, 14840, 16380, 16380, 16200] dup_rows_per_dst=[16, 72, 0, 6308, 4068, 30, 556, 0] combine_rows_per_src=[16384, 16384, 10136, 13318, 14840, 16380, 16380, 16200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129254 payload_rows_remote=113740 dup_rows=1818 dup_rows_remote=1818 dedup_ratio=0.013870 router_tokens_per_expert=[0, 0, 8192, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 58, 8192, 0, 0, 8192, 0, 897, 0, 0, 8192, 0, 7757, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16270, 16384, 16384, 15550, 16384, 16384, 15514, 16384] dup_rows_per_dst=[0, 0, 0, 24, 0, 1794, 0, 0] combine_rows_per_src=[16270, 16384, 16384, 15550, 16384, 16384, 15514, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=934756352 dispatch_meta_bytes=464024 combine_payload_bytes=934756352 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935220376 ingress_bytes=1287077888 link_bytes=1287077888 token_bytes=1869976728 weight_movement_bytes=352321536 total_nvlink_bytes=2222298264 weight_to_token_ratio=0.188410 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.207098920e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=905904128 dispatch_meta_bytes=460768 combine_payload_bytes=905904128 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=906364896 ingress_bytes=1258225664 link_bytes=1258225664 token_bytes=1812269024 weight_movement_bytes=352321536 total_nvlink_bytes=2164590560 weight_to_token_ratio=0.194409 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.045538509e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128750 payload_rows_remote=113032 dup_rows=2322 dup_rows_remote=2018 dedup_ratio=0.017715 router_tokens_per_expert=[0, 1, 8187, 8124, 1, 0, 0, 0, 0, 0, 1, 1, 0, 7747, 1, 263, 8191, 1, 1, 8191, 0, 992, 16, 0, 8191, 4, 7431, 1, 0, 0, 1, 8190] payload_rows_per_dst=[16250, 16384, 16378, 15718, 16384, 16382, 14872, 16382] dup_rows_per_dst=[0, 0, 0, 304, 2, 2016, 0, 0] combine_rows_per_src=[16250, 16384, 16378, 15718, 16384, 16382, 14872, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114566 payload_rows_remote=98182 dup_rows=16506 dup_rows_remote=16476 dedup_ratio=0.125931 router_tokens_per_expert=[0, 0, 15, 1, 3, 0, 0, 1, 0, 64, 5647, 1, 1, 1, 1, 3042, 8143, 8188, 1, 7506, 5, 26, 8190, 0, 0, 0, 0, 167, 1, 8169, 8172, 8191] payload_rows_per_dst=[16384, 16378, 16320, 16376, 1506, 16382, 16344, 14876] dup_rows_per_dst=[30, 6, 10116, 6000, 0, 60, 294, 0] combine_rows_per_src=[16384, 16378, 16320, 16376, 1506, 16382, 16344, 14876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=849035264 dispatch_meta_bytes=458648 combine_payload_bytes=849035264 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=849493912 ingress_bytes=1201356800 link_bytes=1201356800 token_bytes=1698529176 weight_movement_bytes=352321536 total_nvlink_bytes=2050850712 weight_to_token_ratio=0.207427 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.727096609e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=931758080 dispatch_meta_bytes=462232 combine_payload_bytes=931758080 prefetch_bytes=176160768 reduce_bytes=1409286144 egress_bytes=932220312 ingress_bytes=2517204992 link_bytes=2517204992 token_bytes=1863978392 weight_movement_bytes=1585446912 total_nvlink_bytes=3449425304 weight_to_token_ratio=0.850572 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.409529722e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=925958144 dispatch_meta_bytes=460200 combine_payload_bytes=925958144 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=926418344 ingress_bytes=1278279680 link_bytes=1278279680 token_bytes=1852376488 weight_movement_bytes=352321536 total_nvlink_bytes=2204698024 weight_to_token_ratio=0.190200 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.157832628e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128982 payload_rows_remote=112598 dup_rows=2090 dup_rows_remote=2090 dedup_ratio=0.015945 router_tokens_per_expert=[0, 1, 8187, 8153, 1, 0, 0, 0, 0, 0, 1, 1, 0, 7201, 1, 236, 8191, 1, 1, 8191, 0, 965, 10, 0, 8191, 1, 8013, 1, 0, 0, 1, 8188] payload_rows_per_dst=[16308, 16384, 16378, 14738, 16384, 16382, 16030, 16378] dup_rows_per_dst=[0, 0, 0, 138, 2, 1950, 0, 0] combine_rows_per_src=[16308, 16384, 16378, 14738, 16384, 16382, 16030, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117678 payload_rows_remote=101294 dup_rows=13394 dup_rows_remote=13358 dedup_ratio=0.102188 router_tokens_per_expert=[0, 0, 8192, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 0, 34, 8192, 0, 0, 8192, 0, 944, 0, 0, 8192, 0, 7761, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16328, 16376, 16384, 15368, 16368, 4206, 16264, 16384] dup_rows_per_dst=[0, 0, 46, 0, 0, 250, 13062, 36] combine_rows_per_src=[16328, 16376, 16384, 15368, 16368, 4206, 16264, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=804306944 dispatch_meta_bytes=458632 combine_payload_bytes=804306944 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=804765576 ingress_bytes=1156628480 link_bytes=1156628480 token_bytes=1609072520 weight_movement_bytes=352321536 total_nvlink_bytes=1961394056 weight_to_token_ratio=0.218959 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.476636688e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129152 payload_rows_remote=112768 dup_rows=1920 dup_rows_remote=1920 dedup_ratio=0.014648 router_tokens_per_expert=[0, 0, 8192, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7642, 0, 41, 8192, 0, 0, 8192, 0, 953, 0, 0, 8192, 0, 7787, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16306, 16384, 16384, 15352, 16384, 16384, 15574, 16384] dup_rows_per_dst=[0, 0, 0, 14, 0, 1906, 0, 0] combine_rows_per_src=[16306, 16384, 16384, 15352, 16384, 16384, 15574, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128808 payload_rows_remote=113028 dup_rows=2264 dup_rows_remote=2264 dedup_ratio=0.017273 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 7371, 0, 0, 0, 0, 1060, 8176, 8191, 0, 7890, 0, 5, 8192, 0, 0, 0, 0, 78, 0, 8192, 8187, 8192] payload_rows_per_dst=[16384, 16384, 14742, 16374, 15780, 16384, 16380, 16380] dup_rows_per_dst=[0, 0, 2, 2098, 0, 10, 154, 0] combine_rows_per_src=[16384, 16384, 14742, 16374, 15780, 16384, 16380, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=922402816 dispatch_meta_bytes=458752 combine_payload_bytes=922402816 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=922861568 ingress_bytes=1274724352 link_bytes=1274724352 token_bytes=1845264384 weight_movement_bytes=352321536 total_nvlink_bytes=2197585920 weight_to_token_ratio=0.190933 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.137924275e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=829800448 dispatch_meta_bytes=458608 combine_payload_bytes=829800448 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=830259056 ingress_bytes=1182121984 link_bytes=1182121984 token_bytes=1660059504 weight_movement_bytes=352321536 total_nvlink_bytes=2012381040 weight_to_token_ratio=0.212234 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.619389668e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[1, 1, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129136 payload_rows_remote=112756 dup_rows=1936 dup_rows_remote=258 dedup_ratio=0.014771 router_tokens_per_expert=[2, 2, 8181, 8101, 2, 0, 0, 3, 3, 0, 1, 2, 0, 7321, 2, 380, 8189, 3, 2, 8190, 0, 798, 40, 1, 8190, 1, 7930, 2, 0, 0, 2, 8188] payload_rows_per_dst=[16210, 16384, 16374, 15158, 16384, 16380, 15866, 16380] dup_rows_per_dst=[0, 6, 0, 248, 4, 1678, 0, 0] combine_rows_per_src=[16210, 16384, 16374, 15158, 16384, 16380, 15866, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=923795456 dispatch_meta_bytes=458752 combine_payload_bytes=923795456 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=924254208 ingress_bytes=1628438528 link_bytes=1628438528 token_bytes=1848049664 weight_movement_bytes=704643072 total_nvlink_bytes=2552692736 weight_to_token_ratio=0.381290 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.118576013e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000015 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130844 payload_rows_remote=114668 dup_rows=228 dup_rows_remote=228 dedup_ratio=0.001740 router_tokens_per_expert=[8188, 0, 0, 0, 0, 8188, 8191, 0, 1, 1, 8190, 0, 8191, 1, 8087, 8191, 1, 1, 1, 8190, 1, 0, 0, 1, 0, 1, 2, 4, 0, 1, 0, 104] payload_rows_per_dst=[16376, 16376, 16382, 16176, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 0, 4, 2, 12, 208] combine_rows_per_src=[16376, 16376, 16382, 16176, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=925925376 dispatch_meta_bytes=461168 combine_payload_bytes=925925376 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=926386544 ingress_bytes=2335211520 link_bytes=2335211520 token_bytes=1852311920 weight_movement_bytes=1409286144 total_nvlink_bytes=3261598064 weight_to_token_ratio=0.760826 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=23680 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.307620975e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121668 payload_rows_remote=105288 dup_rows=9404 dup_rows_remote=8716 dedup_ratio=0.071747 router_tokens_per_expert=[1, 1, 8178, 8129, 1, 0, 1, 0, 0, 0, 1, 1, 0, 6916, 1, 695, 8190, 1, 1, 8191, 0, 834, 25, 0, 8191, 0, 7996, 1, 0, 0, 0, 8181] payload_rows_per_dst=[16380, 16384, 16360, 15390, 16382, 16382, 8028, 16362] dup_rows_per_dst=[688, 2, 0, 6986, 10, 1718, 0, 0] combine_rows_per_src=[16380, 16384, 16360, 15390, 16382, 16382, 8028, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=923697152 dispatch_meta_bytes=452056 combine_payload_bytes=923697152 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=924149208 ingress_bytes=1276018688 link_bytes=1276018688 token_bytes=1847846360 weight_movement_bytes=352321536 total_nvlink_bytes=2200167896 weight_to_token_ratio=0.190666 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.145172016e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458744 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966456 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474168 weight_movement_bytes=352321536 total_nvlink_bytes=2231795704 weight_to_token_ratio=0.187458 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130914 payload_rows_remote=114538 dup_rows=158 dup_rows_remote=158 dedup_ratio=0.001205 router_tokens_per_expert=[8183, 0, 0, 0, 0, 8188, 8191, 0, 0, 1, 8190, 0, 8191, 1, 8134, 8191, 4, 1, 1, 8183, 1, 0, 0, 1, 0, 1, 2, 13, 0, 1, 1, 57] payload_rows_per_dst=[16366, 16376, 16382, 16270, 16368, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 10, 2, 30, 116] combine_rows_per_src=[16366, 16376, 16382, 16270, 16368, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=459584 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=1233125376 egress_bytes=939819840 ingress_bytes=2348646400 link_bytes=2348646400 token_bytes=1879180096 weight_movement_bytes=1409286144 total_nvlink_bytes=3288466240 weight_to_token_ratio=0.749947 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.315143947e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131030 payload_rows_remote=114646 dup_rows=42 dup_rows_remote=36 dedup_ratio=0.000320 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[16384, 16352, 16380, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 10, 26, 0, 0, 6] combine_rows_per_src=[16384, 16352, 16380, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=862519296 dispatch_meta_bytes=456016 combine_payload_bytes=862519296 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=862975312 ingress_bytes=1919483904 link_bytes=1919483904 token_bytes=1725494608 weight_movement_bytes=1056964608 total_nvlink_bytes=2782459216 weight_to_token_ratio=0.612557 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.074830863e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938295296 dispatch_meta_bytes=458784 combine_payload_bytes=938295296 prefetch_bytes=352321536 reduce_bytes=880803840 egress_bytes=938754080 ingress_bytes=2171420672 link_bytes=2171420672 token_bytes=1877049376 weight_movement_bytes=1233125376 total_nvlink_bytes=3110174752 weight_to_token_ratio=0.656949 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.215904937e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939180032 dispatch_meta_bytes=458728 combine_payload_bytes=939180032 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939638760 ingress_bytes=1291501568 link_bytes=1291501568 token_bytes=1878818792 weight_movement_bytes=352321536 total_nvlink_bytes=2231140328 weight_to_token_ratio=0.187523 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231869682e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130188 payload_rows_remote=113804 dup_rows=884 dup_rows_remote=884 dedup_ratio=0.006744 router_tokens_per_expert=[0, 0, 8192, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 5, 8192, 0, 0, 8192, 0, 437, 1, 0, 8192, 0, 7931, 0, 0, 0, 0, 8191] payload_rows_per_dst=[16332, 16384, 16384, 16076, 16384, 16384, 15862, 16382] dup_rows_per_dst=[0, 0, 0, 8, 0, 876, 0, 0] combine_rows_per_src=[16332, 16384, 16384, 16076, 16384, 16384, 15862, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130812 payload_rows_remote=114428 dup_rows=260 dup_rows_remote=256 dedup_ratio=0.001984 router_tokens_per_expert=[8181, 0, 0, 0, 0, 8182, 8190, 0, 3, 2, 8189, 0, 8189, 2, 8085, 8190, 11, 2, 2, 8186, 2, 0, 0, 2, 0, 2, 6, 4, 0, 2, 0, 104] payload_rows_per_dst=[16362, 16364, 16384, 16174, 16376, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 0, 26, 4, 20, 206] combine_rows_per_src=[16362, 16364, 16384, 16174, 16376, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124766 payload_rows_remote=108382 dup_rows=6306 dup_rows_remote=6306 dedup_ratio=0.048111 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 8192, 5039, 0, 8192, 0, 0, 8192, 0, 0, 0, 3, 0, 3156, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 10078, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 6306, 0, 0] combine_rows_per_src=[16384, 16384, 10078, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939982848 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879506944 weight_movement_bytes=528482304 total_nvlink_bytes=2407989248 weight_to_token_ratio=0.281181 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125408 payload_rows_remote=109030 dup_rows=5664 dup_rows_remote=5664 dedup_ratio=0.043213 router_tokens_per_expert=[1, 1, 4, 1, 12, 0, 0, 8190, 8032, 2, 8191, 5949, 1, 8187, 0, 1, 8191, 0, 1, 1, 4, 0, 2191, 0, 8189, 0, 0, 204, 1, 8180, 1, 1] payload_rows_per_dst=[16384, 16382, 12786, 16378, 16384, 14348, 16382, 16364] dup_rows_per_dst=[12, 22, 1626, 0, 2, 3596, 404, 2] combine_rows_per_src=[16384, 16382, 12786, 16378, 16384, 14348, 16382, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130374 payload_rows_remote=114010 dup_rows=698 dup_rows_remote=630 dedup_ratio=0.005325 router_tokens_per_expert=[8182, 0, 0, 0, 0, 8188, 8191, 0, 1, 1, 8191, 0, 8191, 2, 8104, 8191, 5, 1, 1, 8171, 1, 0, 0, 1, 0, 1, 0, 22, 0, 1, 0, 90] payload_rows_per_dst=[16364, 16382, 16382, 16370, 16344, 16374, 16316, 15842] dup_rows_per_dst=[68, 70, 4, 538, 12, 2, 0, 4] combine_rows_per_src=[16364, 16382, 16382, 16370, 16344, 16374, 16316, 15842] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=932282368 dispatch_meta_bytes=458752 combine_payload_bytes=932282368 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=932741120 ingress_bytes=1813086208 link_bytes=1813086208 token_bytes=1865023488 weight_movement_bytes=880803840 total_nvlink_bytes=2745827328 weight_to_token_ratio=0.472275 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.015252595e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=937394176 dispatch_meta_bytes=458736 combine_payload_bytes=937394176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937852912 ingress_bytes=1289715712 link_bytes=1289715712 token_bytes=1875247088 weight_movement_bytes=352321536 total_nvlink_bytes=2227568624 weight_to_token_ratio=0.187880 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.221869633e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=887865344 dispatch_meta_bytes=458752 combine_payload_bytes=887865344 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=888324096 ingress_bytes=1944829952 link_bytes=1944829952 token_bytes=1776189440 weight_movement_bytes=1056964608 total_nvlink_bytes=2833154048 weight_to_token_ratio=0.595074 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.089023592e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=893173760 dispatch_meta_bytes=458776 combine_payload_bytes=893173760 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=893632536 ingress_bytes=1245495296 link_bytes=1245495296 token_bytes=1786806296 weight_movement_bytes=352321536 total_nvlink_bytes=2139127832 weight_to_token_ratio=0.197179 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.974253762e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119998 payload_rows_remote=103618 dup_rows=11074 dup_rows_remote=11064 dedup_ratio=0.084488 router_tokens_per_expert=[1, 1, 5, 1, 7, 0, 0, 8188, 8026, 2, 8191, 5492, 2, 8186, 0, 2, 8190, 0, 0, 1, 0, 0, 2673, 0, 8190, 0, 1, 196, 0, 8180, 1, 0] payload_rows_per_dst=[16384, 16380, 16382, 16378, 16382, 5346, 16384, 16362] dup_rows_per_dst=[14, 10, 10658, 2, 0, 0, 390, 0] combine_rows_per_src=[16384, 16380, 16382, 16378, 16382, 5346, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121528 payload_rows_remote=105144 dup_rows=9544 dup_rows_remote=9456 dedup_ratio=0.072815 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8186, 8150, 0, 8192, 3482, 0, 8192, 0, 5, 8192, 0, 0, 0, 0, 0, 4753, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16320, 16372, 7006, 16382, 16384, 16384, 16296, 16384] dup_rows_per_dst=[0, 0, 22, 10, 2, 9422, 0, 88] combine_rows_per_src=[16320, 16372, 7006, 16382, 16384, 16384, 16296, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=933969920 dispatch_meta_bytes=458560 combine_payload_bytes=933969920 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934428480 ingress_bytes=1286291456 link_bytes=1286291456 token_bytes=1868398400 weight_movement_bytes=352321536 total_nvlink_bytes=2220719936 weight_to_token_ratio=0.188569 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.202695229e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123276 payload_rows_remote=106894 dup_rows=7796 dup_rows_remote=2 dedup_ratio=0.059479 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8186, 8177, 0, 8192, 3911, 0, 8192, 0, 0, 8192, 0, 0, 0, 2, 0, 4300, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16372, 16382, 16384, 16384, 8602, 16384, 16384] dup_rows_per_dst=[0, 0, 7794, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16372, 16382, 16384, 16384, 8602, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=848838656 dispatch_meta_bytes=458728 combine_payload_bytes=848838656 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=849297384 ingress_bytes=1201160192 link_bytes=1201160192 token_bytes=1698136040 weight_movement_bytes=352321536 total_nvlink_bytes=2050457576 weight_to_token_ratio=0.207475 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.725995686e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=861339648 dispatch_meta_bytes=458400 combine_payload_bytes=861339648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=861798048 ingress_bytes=1213661184 link_bytes=1213661184 token_bytes=1723137696 weight_movement_bytes=352321536 total_nvlink_bytes=2075459232 weight_to_token_ratio=0.204465 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.795996023e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114686 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000046 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 8192, 0, 8191, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16380, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 2] combine_rows_per_src=[16384, 16384, 16384, 16382, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125588 payload_rows_remote=109580 dup_rows=5484 dup_rows_remote=786 dedup_ratio=0.041840 router_tokens_per_expert=[3, 2, 11, 2, 16, 0, 0, 8187, 8001, 5, 8189, 5666, 3, 8180, 0, 2, 8189, 1, 0, 0, 5, 0, 2346, 1, 8186, 0, 0, 362, 5, 8172, 2, 0] payload_rows_per_dst=[16384, 16374, 11340, 16368, 16380, 16008, 16380, 16354] dup_rows_per_dst=[30, 32, 2, 2, 0, 4698, 716, 4] combine_rows_per_src=[16384, 16374, 11340, 16368, 16380, 16008, 16380, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130400 payload_rows_remote=114016 dup_rows=672 dup_rows_remote=672 dedup_ratio=0.005127 router_tokens_per_expert=[0, 1, 335, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 7861, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 15722] dup_rows_per_dst=[672, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 15722] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=875675648 dispatch_meta_bytes=427584 combine_payload_bytes=875675648 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=876103232 ingress_bytes=1932640256 link_bytes=1932640256 token_bytes=1751778880 weight_movement_bytes=1056964608 total_nvlink_bytes=2808743488 weight_to_token_ratio=0.603366 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=20480 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.082197871e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130402 payload_rows_remote=114018 dup_rows=670 dup_rows_remote=668 dedup_ratio=0.005112 router_tokens_per_expert=[0, 0, 250, 1, 1, 0, 81, 1, 8182, 8191, 0, 0, 1, 0, 0, 1, 8187, 8191, 0, 1, 1, 8189, 0, 2, 0, 0, 1, 8191, 7889, 4, 46, 8125] payload_rows_per_dst=[16256, 16384, 16364, 16384, 16376, 16384, 16384, 15870] dup_rows_per_dst=[496, 164, 0, 2, 0, 0, 0, 8] combine_rows_per_src=[16256, 16384, 16364, 16384, 16376, 16384, 16384, 15870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118812 payload_rows_remote=102428 dup_rows=12260 dup_rows_remote=12252 dedup_ratio=0.093536 router_tokens_per_expert=[1, 0, 3, 1, 10, 0, 3, 8161, 8073, 1, 8191, 5318, 1, 8185, 0, 18, 8182, 1, 0, 1, 7, 0, 2848, 0, 8189, 0, 0, 161, 4, 8175, 1, 1] payload_rows_per_dst=[16384, 16336, 16382, 16372, 16366, 5696, 14916, 16360] dup_rows_per_dst=[8, 48, 12092, 22, 22, 14, 52, 2] combine_rows_per_src=[16384, 16336, 16382, 16372, 16366, 5696, 14916, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458768 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966480 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474192 weight_movement_bytes=352321536 total_nvlink_bytes=2231795728 weight_to_token_ratio=0.187458 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=897679360 dispatch_meta_bytes=441464 combine_payload_bytes=897679360 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=898120824 ingress_bytes=1250000896 link_bytes=1250000896 token_bytes=1795800184 weight_movement_bytes=352321536 total_nvlink_bytes=2148121720 weight_to_token_ratio=0.196192 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=20480 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.999483242e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=934019072 dispatch_meta_bytes=458752 combine_payload_bytes=934019072 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934477824 ingress_bytes=1286340608 link_bytes=1286340608 token_bytes=1868496896 weight_movement_bytes=352321536 total_nvlink_bytes=2220818432 weight_to_token_ratio=0.188559 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.202970460e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129752 payload_rows_remote=113368 dup_rows=1320 dup_rows_remote=1150 dedup_ratio=0.010071 router_tokens_per_expert=[0, 0, 571, 1, 1, 0, 84, 1, 8181, 8191, 0, 0, 1, 0, 0, 1, 8188, 8191, 0, 1, 1, 8189, 0, 3, 0, 0, 1, 8191, 7573, 2, 46, 8118] payload_rows_per_dst=[16238, 16384, 16362, 16384, 16378, 16384, 16384, 15238] dup_rows_per_dst=[1142, 170, 0, 2, 0, 2, 0, 4] combine_rows_per_src=[16238, 16384, 16362, 16384, 16378, 16384, 16384, 15238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=934035456 dispatch_meta_bytes=458744 combine_payload_bytes=934035456 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934494200 ingress_bytes=1286356992 link_bytes=1286356992 token_bytes=1868529656 weight_movement_bytes=352321536 total_nvlink_bytes=2220851192 weight_to_token_ratio=0.188555 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.203062203e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119682 payload_rows_remote=103880 dup_rows=11390 dup_rows_remote=1010 dedup_ratio=0.086899 router_tokens_per_expert=[0, 0, 799, 0, 0, 0, 0, 0, 8146, 8192, 1, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 7445, 0, 0, 8185] payload_rows_per_dst=[6438, 15624, 16294, 16376, 16384, 16380, 16384, 15802] dup_rows_per_dst=[672, 0, 290, 0, 44, 0, 4, 10380] combine_rows_per_src=[6438, 15624, 16294, 16376, 16384, 16380, 16384, 15802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=839090176 dispatch_meta_bytes=458720 combine_payload_bytes=839090176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=839548896 ingress_bytes=1191411712 link_bytes=1191411712 token_bytes=1678639072 weight_movement_bytes=352321536 total_nvlink_bytes=2030960608 weight_to_token_ratio=0.209885 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.671408267e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127264 payload_rows_remote=110890 dup_rows=3808 dup_rows_remote=3808 dedup_ratio=0.029053 router_tokens_per_expert=[0, 0, 1908, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 6293, 0, 0, 8188] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 12586] dup_rows_per_dst=[3808, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 12586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=928710656 dispatch_meta_bytes=458072 combine_payload_bytes=928710656 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=929168728 ingress_bytes=1281032192 link_bytes=1281032192 token_bytes=1857879384 weight_movement_bytes=352321536 total_nvlink_bytes=2210200920 weight_to_token_ratio=0.189636 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.173245546e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=850984960 dispatch_meta_bytes=419560 combine_payload_bytes=850984960 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=851404520 ingress_bytes=1555628032 link_bytes=1555628032 token_bytes=1702389480 weight_movement_bytes=704643072 total_nvlink_bytes=2407032552 weight_to_token_ratio=0.413914 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.710867628e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=908410880 dispatch_meta_bytes=458792 combine_payload_bytes=908410880 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=908869672 ingress_bytes=2141536256 link_bytes=2141536256 token_bytes=1817280552 weight_movement_bytes=1233125376 total_nvlink_bytes=3050405928 weight_to_token_ratio=0.678555 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.199170911e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130472 payload_rows_remote=114088 dup_rows=600 dup_rows_remote=0 dedup_ratio=0.004578 router_tokens_per_expert=[0, 0, 0, 0, 7894, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 8192, 300, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 15788, 16384, 16382, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 600, 0] combine_rows_per_src=[16384, 15788, 16384, 16382, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129478 payload_rows_remote=113094 dup_rows=1594 dup_rows_remote=1582 dedup_ratio=0.012161 router_tokens_per_expert=[3, 1, 634, 3, 2, 0, 156, 2, 8184, 8190, 0, 0, 2, 0, 0, 3, 8180, 8188, 0, 3, 2, 8188, 0, 8, 0, 0, 2, 8189, 7483, 1, 77, 8035] payload_rows_per_dst=[16092, 16384, 16368, 16382, 16366, 16384, 16382, 15120] dup_rows_per_dst=[1260, 316, 0, 4, 0, 12, 0, 2] combine_rows_per_src=[16092, 16384, 16368, 16382, 16366, 16384, 16382, 15120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123184 payload_rows_remote=106802 dup_rows=7888 dup_rows_remote=7888 dedup_ratio=0.060181 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 8179, 8176, 0, 8192, 4261, 0, 8191, 0, 4, 8191, 0, 0, 0, 4, 0, 3953, 0, 8192, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16360, 8522, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 6, 0, 7882, 0, 0] combine_rows_per_src=[16384, 16360, 8522, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130592 payload_rows_remote=114210 dup_rows=480 dup_rows_remote=476 dedup_ratio=0.003662 router_tokens_per_expert=[0, 0, 1, 0, 8093, 1, 1, 8190, 1, 1, 1, 1, 8191, 2, 0, 0, 8189, 0, 133, 8190, 1, 0, 1, 8191, 101, 0, 0, 0, 1, 8056, 0, 8190] payload_rows_per_dst=[16380, 16188, 16384, 16382, 16380, 16384, 16380, 16114] dup_rows_per_dst=[0, 2, 4, 4, 266, 2, 202, 0] combine_rows_per_src=[16380, 16188, 16384, 16382, 16380, 16384, 16380, 16114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129856 payload_rows_remote=113552 dup_rows=1216 dup_rows_remote=124 dedup_ratio=0.009277 router_tokens_per_expert=[0, 0, 565, 1, 1, 0, 54, 1, 8138, 8191, 1, 0, 1, 0, 0, 1, 8185, 8191, 0, 1, 1, 8191, 0, 1, 0, 0, 1, 8190, 7656, 4, 29, 8132] payload_rows_per_dst=[16304, 16384, 16288, 16384, 16372, 16372, 16382, 15370] dup_rows_per_dst=[1092, 110, 2, 2, 0, 2, 0, 8] combine_rows_per_src=[16304, 16384, 16288, 16384, 16372, 16372, 16382, 15370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=934608896 dispatch_meta_bytes=456352 combine_payload_bytes=934608896 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=935065248 ingress_bytes=1286930432 link_bytes=1286930432 token_bytes=1869674144 weight_movement_bytes=352321536 total_nvlink_bytes=2221995680 weight_to_token_ratio=0.188440 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.206273228e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=926466048 dispatch_meta_bytes=458704 combine_payload_bytes=926466048 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=926924752 ingress_bytes=1454948352 link_bytes=1454948352 token_bytes=1853390800 weight_movement_bytes=528482304 total_nvlink_bytes=2381873104 weight_to_token_ratio=0.285143 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.147103446e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130712 payload_rows_remote=114426 dup_rows=360 dup_rows_remote=358 dedup_ratio=0.002747 router_tokens_per_expert=[0, 0, 1, 0, 8142, 1, 1, 8190, 1, 1, 1, 1, 8191, 1, 0, 0, 8189, 0, 119, 8185, 1, 0, 2, 8191, 55, 0, 0, 0, 1, 8072, 0, 8190] payload_rows_per_dst=[16380, 16286, 16384, 16382, 16370, 16384, 16380, 16146] dup_rows_per_dst=[0, 2, 4, 2, 238, 4, 110, 0] combine_rows_per_src=[16380, 16286, 16384, 16382, 16370, 16384, 16380, 16146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=874921984 dispatch_meta_bytes=458760 combine_payload_bytes=874921984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=875380744 ingress_bytes=1227243520 link_bytes=1227243520 token_bytes=1750302728 weight_movement_bytes=352321536 total_nvlink_bytes=2102624264 weight_to_token_ratio=0.201292 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.872051435e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=935608320 dispatch_meta_bytes=458744 combine_payload_bytes=935608320 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=936067064 ingress_bytes=1287929856 link_bytes=1287929856 token_bytes=1871675384 weight_movement_bytes=352321536 total_nvlink_bytes=2223996920 weight_to_token_ratio=0.188239 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.211869585e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129516 payload_rows_remote=113132 dup_rows=1556 dup_rows_remote=270 dedup_ratio=0.011871 router_tokens_per_expert=[0, 0, 0, 0, 8060, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 8192, 133, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16098, 16378, 16382, 16020, 16384, 15486, 16384] dup_rows_per_dst=[0, 0, 0, 4, 0, 2, 264, 1286] combine_rows_per_src=[16384, 16098, 16378, 16382, 16020, 16384, 15486, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130812 payload_rows_remote=114428 dup_rows=260 dup_rows_remote=260 dedup_ratio=0.001984 router_tokens_per_expert=[0, 0, 0, 0, 8062, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1, 8192, 0, 0, 0, 8192, 129, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16124, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 258, 0] combine_rows_per_src=[16384, 16124, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=930217984 dispatch_meta_bytes=454704 combine_payload_bytes=930217984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930672688 ingress_bytes=1282539520 link_bytes=1282539520 token_bytes=1860890672 weight_movement_bytes=352321536 total_nvlink_bytes=2213212208 weight_to_token_ratio=0.189330 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.181685954e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937377792 dispatch_meta_bytes=459136 combine_payload_bytes=937377792 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=937836928 ingress_bytes=1994342400 link_bytes=1994342400 token_bytes=1875214720 weight_movement_bytes=1056964608 total_nvlink_bytes=2932179328 weight_to_token_ratio=0.563650 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.116748496e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937394176 dispatch_meta_bytes=458752 combine_payload_bytes=937394176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937852928 ingress_bytes=1289715712 link_bytes=1289715712 token_bytes=1875247104 weight_movement_bytes=352321536 total_nvlink_bytes=2227568640 weight_to_token_ratio=0.187880 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.221869633e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=926777344 dispatch_meta_bytes=453608 combine_payload_bytes=926777344 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=927230952 ingress_bytes=1631420416 link_bytes=1631420416 token_bytes=1854008296 weight_movement_bytes=704643072 total_nvlink_bytes=2558651368 weight_to_token_ratio=0.380065 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.135273341e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114678 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[8191, 8188, 2, 8192, 0, 0, 0, 8191, 0, 0, 8192, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16382, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[2, 0, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 16382, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130352 payload_rows_remote=113968 dup_rows=720 dup_rows_remote=716 dedup_ratio=0.005493 router_tokens_per_expert=[3, 0, 2, 0, 8094, 2, 2, 8187, 0, 4, 3, 2, 8190, 2, 0, 0, 8186, 0, 252, 8188, 2, 0, 3, 8189, 96, 0, 0, 0, 2, 7938, 0, 8189] payload_rows_per_dst=[16378, 16192, 16384, 16380, 16376, 16384, 16378, 15880] dup_rows_per_dst=[4, 4, 8, 4, 504, 4, 192, 0] combine_rows_per_src=[16378, 16192, 16384, 16380, 16376, 16384, 16378, 15880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130934 payload_rows_remote=114552 dup_rows=138 dup_rows_remote=138 dedup_ratio=0.001053 router_tokens_per_expert=[8189, 8182, 15, 8184, 2, 2, 1, 8191, 1, 0, 8190, 57, 1, 8190, 0, 0, 0, 1, 0, 0, 1, 1, 8135, 0, 0, 0, 8191, 1, 1, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 16382, 16382, 16370, 16272, 16384, 16380] dup_rows_per_dst=[14, 8, 114, 0, 0, 2, 0, 0] combine_rows_per_src=[16380, 16384, 16382, 16382, 16370, 16272, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129622 payload_rows_remote=113238 dup_rows=1450 dup_rows_remote=1450 dedup_ratio=0.011063 router_tokens_per_expert=[0, 0, 725, 0, 0, 0, 0, 0, 8178, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 7481, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16356, 16384, 16384, 16384, 16384, 14962] dup_rows_per_dst=[1450, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16356, 16384, 16384, 16384, 16384, 14962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128636 payload_rows_remote=112252 dup_rows=2436 dup_rows_remote=2426 dedup_ratio=0.018585 router_tokens_per_expert=[0, 0, 1, 0, 8087, 1, 1, 8190, 1, 1, 1, 1, 8190, 1, 0, 0, 8190, 0, 137, 8176, 1, 0, 1, 8191, 119, 0, 0, 0, 1, 8054, 0, 8191] payload_rows_per_dst=[16384, 16314, 16368, 16382, 15498, 16372, 15208, 16110] dup_rows_per_dst=[10, 1910, 6, 0, 272, 0, 238, 0] combine_rows_per_src=[16384, 16314, 16368, 16382, 15498, 16372, 15208, 16110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130890 payload_rows_remote=114506 dup_rows=182 dup_rows_remote=176 dedup_ratio=0.001389 router_tokens_per_expert=[8186, 8170, 9, 8184, 2, 2, 1, 8190, 1, 0, 8190, 78, 1, 8191, 0, 0, 0, 1, 0, 0, 3, 1, 8133, 0, 0, 0, 8191, 1, 1, 0, 0, 0] payload_rows_per_dst=[16344, 16384, 16382, 16384, 16370, 16268, 16384, 16374] dup_rows_per_dst=[14, 6, 156, 0, 0, 6, 0, 0] combine_rows_per_src=[16344, 16384, 16382, 16384, 16370, 16268, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458752 combine_payload_bytes=939442176 prefetch_bytes=0 reduce_bytes=0 egress_bytes=939900928 ingress_bytes=939442176 link_bytes=939900928 token_bytes=1879343104 weight_movement_bytes=0 total_nvlink_bytes=1879343104 weight_to_token_ratio=0.000000 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.026515 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.263052863e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=933625856 dispatch_meta_bytes=458736 combine_payload_bytes=933625856 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=934084592 ingress_bytes=1462108160 link_bytes=1462108160 token_bytes=1867710448 weight_movement_bytes=528482304 total_nvlink_bytes=2396192752 weight_to_token_ratio=0.282957 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.187195382e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938409984 dispatch_meta_bytes=458760 combine_payload_bytes=938409984 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938868744 ingress_bytes=1290731520 link_bytes=1290731520 token_bytes=1877278728 weight_movement_bytes=352321536 total_nvlink_bytes=2229600264 weight_to_token_ratio=0.187677 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227557734e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=927645696 dispatch_meta_bytes=458752 combine_payload_bytes=927645696 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=928104448 ingress_bytes=1632288768 link_bytes=1632288768 token_bytes=1855750144 weight_movement_bytes=704643072 total_nvlink_bytes=2560393216 weight_to_token_ratio=0.379708 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.140135750e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131024 payload_rows_remote=114640 dup_rows=48 dup_rows_remote=0 dedup_ratio=0.000366 router_tokens_per_expert=[8192, 8178, 1, 8192, 0, 0, 0, 8181, 0, 0, 8192, 24, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16358, 16362, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 48, 0, 0, 0, 0, 0] combine_rows_per_src=[16358, 16362, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130374 payload_rows_remote=113994 dup_rows=698 dup_rows_remote=584 dedup_ratio=0.005325 router_tokens_per_expert=[8190, 8163, 3, 8192, 0, 0, 0, 8186, 0, 0, 8192, 35, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16146, 16366, 15948, 16384, 16384, 16382, 16384, 16380] dup_rows_per_dst=[2, 0, 70, 6, 70, 436, 0, 114] combine_rows_per_src=[16146, 16366, 15948, 16384, 16384, 16382, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=919568384 dispatch_meta_bytes=458712 combine_payload_bytes=919568384 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=920027096 ingress_bytes=1624211456 link_bytes=1624211456 token_bytes=1839595480 weight_movement_bytes=704643072 total_nvlink_bytes=2544238552 weight_to_token_ratio=0.383042 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.094906174e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28125367800 prefetch_bytes=4051697664 reduce_bytes=5460983808 weight_movement_bytes=9512681472 egress_bytes=14066355704 ingress_bytes=23571693568 link_bytes=23572152320 total_nvlink_bytes=37638049272 weight_to_token_ratio=0.338224 prefetch_share_of_ingress=0.171888 dedup_ratio=0.063560 prefetch_slots=46 unused_prefetch_slots=210 slot_utilization=0.179688 reduce_slots=31 remote_demand=23 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.022372 gemm_inner_pad_upper_bound=17399 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.249638343e-05 prefetch_bytes_per_gemm_flop=1.417988479e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938033152 dispatch_meta_bytes=458728 combine_payload_bytes=938033152 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938491880 ingress_bytes=1466515456 link_bytes=1466515456 token_bytes=1876525032 weight_movement_bytes=528482304 total_nvlink_bytes=2405007336 weight_to_token_ratio=0.281628 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.211874400e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28723986520 prefetch_bytes=4404019200 reduce_bytes=4227858432 weight_movement_bytes=8631877632 egress_bytes=14365655128 ingress_bytes=22990209024 link_bytes=22990209024 total_nvlink_bytes=37355864152 weight_to_token_ratio=0.300511 prefetch_share_of_ingress=0.191561 dedup_ratio=0.043330 prefetch_slots=50 unused_prefetch_slots=206 slot_utilization=0.195312 reduce_slots=24 remote_demand=25 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020478 gemm_inner_pad_upper_bound=19812 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.045973372e-05 prefetch_bytes_per_gemm_flop=1.541291824e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=458560 combine_payload_bytes=939130880 prefetch_bytes=176160768 reduce_bytes=1056964608 egress_bytes=939589440 ingress_bytes=2172256256 link_bytes=2172256256 token_bytes=1878720320 weight_movement_bytes=1233125376 total_nvlink_bytes=3111845696 weight_to_token_ratio=0.656365 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216372829e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=933838848 dispatch_meta_bytes=458312 combine_payload_bytes=933838848 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=934297160 ingress_bytes=1638481920 link_bytes=1638481920 token_bytes=1868136008 weight_movement_bytes=704643072 total_nvlink_bytes=2572779080 weight_to_token_ratio=0.377190 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.174814816e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27861754720 prefetch_bytes=5460983808 reduce_bytes=4756340736 weight_movement_bytes=10217324544 egress_bytes=13934486368 ingress_bytes=24144592896 link_bytes=24144592896 total_nvlink_bytes=38079079264 weight_to_token_ratio=0.366715 prefetch_share_of_ingress=0.226178 dedup_ratio=0.068102 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=27 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019531 gemm_inner_pad_upper_bound=20447 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.449977611e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130826 payload_rows_remote=114626 dup_rows=246 dup_rows_remote=244 dedup_ratio=0.001877 router_tokens_per_expert=[8187, 8179, 16, 8176, 3, 5, 4, 8189, 2, 0, 8185, 100, 3, 8188, 0, 0, 0, 2, 3, 0, 1, 3, 8097, 0, 0, 0, 8190, 2, 1, 0, 0, 0] payload_rows_per_dst=[16362, 16384, 16378, 16382, 16360, 16200, 16384, 16376] dup_rows_per_dst=[28, 18, 196, 0, 2, 2, 0, 0] combine_rows_per_src=[16362, 16384, 16378, 16382, 16360, 16200, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28554991608 prefetch_bytes=4932501504 reduce_bytes=10041163776 weight_movement_bytes=14973665280 egress_bytes=14281136120 ingress_bytes=29247520768 link_bytes=29247520768 total_nvlink_bytes=43528656888 weight_to_token_ratio=0.524380 prefetch_share_of_ingress=0.168647 dedup_ratio=0.047906 prefetch_slots=56 unused_prefetch_slots=200 slot_utilization=0.218750 reduce_slots=57 remote_demand=28 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019058 gemm_inner_pad_upper_bound=21336 gemm_flops_step=285735584268288 bytes_per_gemm_flop=1.023586924e-04 prefetch_bytes_per_gemm_flop=1.726246843e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130848 payload_rows_remote=114468 dup_rows=224 dup_rows_remote=224 dedup_ratio=0.001709 router_tokens_per_expert=[0, 0, 0, 0, 8082, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 8192, 112, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16164, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 224, 0] combine_rows_per_src=[16384, 16164, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130718 payload_rows_remote=114336 dup_rows=354 dup_rows_remote=44 dedup_ratio=0.002701 router_tokens_per_expert=[8154, 8124, 14, 8181, 1, 2, 1, 8191, 1, 0, 8187, 120, 1, 8190, 0, 0, 0, 1, 0, 0, 13, 1, 8161, 0, 0, 0, 8191, 1, 1, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16188, 16382, 16364, 16324, 16384, 16310] dup_rows_per_dst=[310, 18, 0, 0, 0, 26, 0, 0] combine_rows_per_src=[16382, 16384, 16188, 16382, 16364, 16324, 16384, 16310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=26931248216 prefetch_bytes=5989466112 reduce_bytes=4404019200 weight_movement_bytes=10393485312 egress_bytes=13469154392 ingress_bytes=23855579136 link_bytes=23855579136 total_nvlink_bytes=37324733528 weight_to_token_ratio=0.385927 prefetch_share_of_ingress=0.251072 dedup_ratio=0.092476 prefetch_slots=68 unused_prefetch_slots=188 slot_utilization=0.265625 reduce_slots=25 remote_demand=34 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=19304 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.348830335e-05 prefetch_bytes_per_gemm_flop=2.096156881e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939016192 dispatch_meta_bytes=459480 combine_payload_bytes=939016192 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939475672 ingress_bytes=1291337728 link_bytes=1291337728 token_bytes=1878491864 weight_movement_bytes=352321536 total_nvlink_bytes=2230813400 weight_to_token_ratio=0.187556 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230952246e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[4, 1, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937721856 dispatch_meta_bytes=458768 combine_payload_bytes=937721856 prefetch_bytes=352321536 reduce_bytes=1233125376 egress_bytes=938180624 ingress_bytes=2523168768 link_bytes=2523168768 token_bytes=1875902480 weight_movement_bytes=1585446912 total_nvlink_bytes=3461349392 weight_to_token_ratio=0.845165 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.412869188e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=936640512 dispatch_meta_bytes=457520 combine_payload_bytes=936640512 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=937098032 ingress_bytes=2698248192 link_bytes=2698248192 token_bytes=1873738544 weight_movement_bytes=1761607680 total_nvlink_bytes=3635346224 weight_to_token_ratio=0.940157 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.510906357e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28342537016 prefetch_bytes=4580179968 reduce_bytes=1937768448 weight_movement_bytes=6517948416 egress_bytes=14174931768 ingress_bytes=20685553664 link_bytes=20685553664 total_nvlink_bytes=34860485432 weight_to_token_ratio=0.229971 prefetch_share_of_ingress=0.221419 dedup_ratio=0.053456 prefetch_slots=52 unused_prefetch_slots=204 slot_utilization=0.203125 reduce_slots=11 remote_demand=26 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020478 gemm_inner_pad_upper_bound=19050 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.239404121e-05 prefetch_bytes_per_gemm_flop=1.602943497e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=26779560384 prefetch_bytes=5637144576 reduce_bytes=4404019200 weight_movement_bytes=10041163776 egress_bytes=13393373632 ingress_bytes=23427350528 link_bytes=23427350528 total_nvlink_bytes=36820724160 weight_to_token_ratio=0.374956 prefetch_share_of_ingress=0.240622 dedup_ratio=0.097760 prefetch_slots=64 unused_prefetch_slots=192 slot_utilization=0.250000 reduce_slots=25 remote_demand=32 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020123 gemm_inner_pad_upper_bound=20193 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.198961494e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=131020 payload_rows_remote=114636 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.000397 router_tokens_per_expert=[8186, 8163, 13, 8192, 0, 0, 0, 8188, 0, 0, 8192, 25, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16352, 16376, 16384, 16384, 16384, 16384, 16384, 16372] dup_rows_per_dst=[0, 0, 50, 0, 0, 2, 0, 0] combine_rows_per_src=[16352, 16376, 16384, 16384, 16384, 16384, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939098112 dispatch_meta_bytes=458752 combine_payload_bytes=939098112 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=939556864 ingress_bytes=1643741184 link_bytes=1643741184 token_bytes=1878654976 weight_movement_bytes=704643072 total_nvlink_bytes=2583298048 weight_to_token_ratio=0.375078 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204264499e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28865028864 prefetch_bytes=5989466112 reduce_bytes=5813305344 weight_movement_bytes=11802771456 egress_bytes=14436180736 ingress_bytes=26231619584 link_bytes=26231619584 total_nvlink_bytes=40667800320 weight_to_token_ratio=0.408895 prefetch_share_of_ingress=0.228330 dedup_ratio=0.038008 prefetch_slots=68 unused_prefetch_slots=188 slot_utilization=0.265625 reduce_slots=33 remote_demand=34 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019176 gemm_inner_pad_upper_bound=20828 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.180382503e-05 prefetch_bytes_per_gemm_flop=2.096156881e-05 anomalies=none - [2026-09-16 21:37:09] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 897.7 | throughput per GPU (TFLOP/s/GPU): 371.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.972642E+00 | loss scale: 1.0 | grad norm: 5.216 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=94206 payload_rows_remote=81544 dup_rows=36866 dup_rows_remote=31252 dedup_ratio=0.281265 router_tokens_per_expert=[3561, 2, 1298, 4827, 2457, 1, 2110, 4570, 824, 4788, 3, 2264, 1592, 2029, 3482, 594, 2, 2, 450, 1, 0, 2790, 2134, 4715, 2, 783, 6075, 1720, 1715, 3231, 2391, 5123] payload_rows_per_dst=[15106, 12662, 11902, 11284, 13476, 8854, 13294, 7628] dup_rows_per_dst=[13700, 5614, 3856, 4110, 4142, 994, 3866, 584] combine_rows_per_src=[15106, 12662, 11902, 11284, 13476, 8854, 13294, 7628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=99190 payload_rows_remote=87316 dup_rows=31882 dup_rows_remote=26728 dedup_ratio=0.243240 router_tokens_per_expert=[4101, 1, 1298, 4316, 1847, 2, 1971, 5087, 894, 5677, 0, 2482, 1389, 2996, 3776, 353, 5, 0, 682, 0, 4, 2967, 1777, 3757, 3, 795, 6096, 1258, 1230, 3657, 2329, 4786] payload_rows_per_dst=[9536, 12942, 13360, 11874, 13684, 11920, 13104, 12770] dup_rows_per_dst=[1836, 4872, 4746, 5154, 4576, 4518, 3200, 2980] combine_rows_per_src=[9536, 12942, 13360, 11874, 13684, 11920, 13104, 12770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=96212 payload_rows_remote=83438 dup_rows=34860 dup_rows_remote=31500 dedup_ratio=0.265961 router_tokens_per_expert=[3644, 0, 1550, 4533, 1842, 0, 2559, 4710, 788, 4872, 2, 1832, 1624, 2528, 3228, 676, 0, 18, 1039, 0, 3, 3282, 2177, 4289, 7, 827, 5856, 1377, 1423, 3429, 2998, 4423] payload_rows_per_dst=[9046, 12618, 11602, 11564, 10072, 13418, 12774, 15118] dup_rows_per_dst=[1342, 5078, 3386, 4494, 888, 6664, 3360, 9648] combine_rows_per_src=[9046, 12618, 11602, 11564, 10072, 13418, 12774, 15118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98408 payload_rows_remote=85684 dup_rows=32664 dup_rows_remote=29490 dedup_ratio=0.249207 router_tokens_per_expert=[4142, 7, 1217, 3888, 1952, 0, 1580, 4556, 554, 5534, 2, 1859, 1434, 3616, 4474, 606, 1, 12, 993, 0, 1, 3397, 1900, 3821, 7, 811, 4985, 1058, 1648, 4053, 3137, 4291] payload_rows_per_dst=[13258, 12376, 12724, 13282, 13892, 9336, 11448, 12092] dup_rows_per_dst=[6760, 3800, 3174, 6978, 4808, 1620, 2274, 3250] combine_rows_per_src=[13258, 12376, 12724, 13282, 13892, 9336, 11448, 12092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=93638 payload_rows_remote=78314 dup_rows=37434 dup_rows_remote=27256 dedup_ratio=0.285599 router_tokens_per_expert=[3602, 1, 1311, 4628, 1806, 4, 2671, 3694, 580, 5650, 0, 2006, 1649, 3109, 4013, 505, 13, 8, 492, 0, 0, 2834, 1811, 4260, 18, 904, 5707, 1488, 1641, 3702, 2780, 4649] payload_rows_per_dst=[8674, 8300, 13566, 10608, 9800, 14350, 13016, 15324] dup_rows_per_dst=[1154, 1348, 5670, 4074, 524, 9874, 4612, 10178] combine_rows_per_src=[8674, 8300, 13566, 10608, 9800, 14350, 13016, 15324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=92000 payload_rows_remote=76636 dup_rows=39072 dup_rows_remote=23178 dedup_ratio=0.298096 router_tokens_per_expert=[3517, 0, 2148, 5011, 2299, 9, 1459, 4452, 586, 4211, 2, 1722, 1252, 3213, 3237, 498, 18, 8, 1575, 1, 2, 3203, 2201, 4953, 15, 640, 6471, 2137, 597, 3676, 1918, 4505] payload_rows_per_dst=[15364, 12250, 11584, 10320, 14132, 9492, 14028, 4830] dup_rows_per_dst=[15894, 4592, 4108, 3026, 5434, 1320, 4498, 200] combine_rows_per_src=[15364, 12250, 11584, 10320, 14132, 9492, 14028, 4830] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97542 payload_rows_remote=84086 dup_rows=33530 dup_rows_remote=27828 dedup_ratio=0.255814 router_tokens_per_expert=[3265, 0, 1559, 4433, 2479, 2, 2063, 3899, 664, 5383, 1, 1749, 1724, 2194, 3652, 457, 4, 3, 956, 1, 0, 2752, 2092, 4735, 0, 808, 5892, 1603, 1631, 4286, 2377, 4872] payload_rows_per_dst=[8424, 12050, 12200, 11528, 13040, 13456, 12972, 13872] dup_rows_per_dst=[1224, 4836, 3394, 4526, 4414, 5702, 3634, 5800] combine_rows_per_src=[8424, 12050, 12200, 11528, 13040, 13456, 12972, 13872] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=98468 payload_rows_remote=84902 dup_rows=32604 dup_rows_remote=27980 dedup_ratio=0.248749 router_tokens_per_expert=[3308, 2, 1567, 4420, 1857, 13, 2026, 4971, 973, 5081, 0, 1846, 1683, 2707, 3891, 844, 1, 5, 1013, 0, 2, 3349, 1630, 4796, 5, 579, 5767, 1145, 1348, 3133, 2631, 4943] payload_rows_per_dst=[8462, 12840, 12128, 12510, 13566, 13602, 12266, 13094] dup_rows_per_dst=[1292, 4894, 3672, 5740, 4624, 5952, 2726, 3704] combine_rows_per_src=[8462, 12840, 12128, 12510, 13566, 13602, 12266, 13094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=668008448 dispatch_meta_bytes=451184 combine_payload_bytes=668008448 prefetch_bytes=352321536 reduce_bytes=528482304 egress_bytes=668459632 ingress_bytes=1548812288 link_bytes=1548812288 token_bytes=1336468080 weight_movement_bytes=880803840 total_nvlink_bytes=2217271920 weight_to_token_ratio=0.659053 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=16640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.672702307e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=715292672 dispatch_meta_bytes=456176 combine_payload_bytes=715292672 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=715748848 ingress_bytes=1596096512 link_bytes=1596096512 token_bytes=1431041520 weight_movement_bytes=880803840 total_nvlink_bytes=2311845360 weight_to_token_ratio=0.615498 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.937474224e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=683524096 dispatch_meta_bytes=459752 combine_payload_bytes=683524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=683983848 ingress_bytes=1035845632 link_bytes=1035845632 token_bytes=1367507944 weight_movement_bytes=352321536 total_nvlink_bytes=1719829480 weight_to_token_ratio=0.257638 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=23552 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.800303156e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=701923328 dispatch_meta_bytes=460696 combine_payload_bytes=701923328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=702384024 ingress_bytes=1054244864 link_bytes=1054244864 token_bytes=1404307352 weight_movement_bytes=352321536 total_nvlink_bytes=1756628888 weight_to_token_ratio=0.250886 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=20736 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.903331175e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=641548288 dispatch_meta_bytes=422280 combine_payload_bytes=641548288 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=641970568 ingress_bytes=1698512896 link_bytes=1698512896 token_bytes=1283518856 weight_movement_bytes=1056964608 total_nvlink_bytes=2340483464 weight_to_token_ratio=0.823490 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.024621 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.510963223e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=627802112 dispatch_meta_bytes=399256 combine_payload_bytes=627802112 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=628201368 ingress_bytes=1332445184 link_bytes=1332445184 token_bytes=1256003480 weight_movement_bytes=704643072 total_nvlink_bytes=1960646552 weight_to_token_ratio=0.561020 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=15232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.461136840e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=688832512 dispatch_meta_bytes=447656 combine_payload_bytes=688832512 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=689280168 ingress_bytes=1393475584 link_bytes=1393475584 token_bytes=1378112680 weight_movement_bytes=704643072 total_nvlink_bytes=2082755752 weight_to_token_ratio=0.511310 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=12544 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.802881605e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=695517184 dispatch_meta_bytes=451528 combine_payload_bytes=695517184 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=695968712 ingress_bytes=1400160256 link_bytes=1400160256 token_bytes=1391485896 weight_movement_bytes=704643072 total_nvlink_bytes=2096128968 weight_to_token_ratio=0.506396 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.015152 gemm_active_groups=14 gemm_peak_group_rows=19200 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.840312978e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114224 payload_rows_remote=98964 dup_rows=16848 dup_rows_remote=11682 dedup_ratio=0.128540 router_tokens_per_expert=[1272, 0, 12, 8187, 3811, 1, 0, 0, 355, 7838, 8183, 8190, 3426, 7934, 49, 16, 8191, 0, 1, 1, 1, 1, 0, 0, 0, 37, 6973, 0, 1, 871, 0, 185] payload_rows_per_dst=[11024, 7624, 15240, 15942, 16384, 16382, 15260, 16368] dup_rows_per_dst=[1512, 0, 676, 7378, 2, 2, 5166, 2112] combine_rows_per_src=[11024, 7624, 15240, 15942, 16384, 16382, 15260, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114498 payload_rows_remote=109610 dup_rows=16574 dup_rows_remote=16420 dedup_ratio=0.126450 router_tokens_per_expert=[2017, 0, 2, 8180, 5207, 3, 3, 2, 1133, 7977, 8170, 8187, 2267, 7815, 108, 2, 8187, 0, 0, 2, 0, 0, 0, 1, 2, 2, 5961, 0, 0, 238, 5, 65] payload_rows_per_dst=[16380, 16206, 15992, 4888, 16378, 16376, 11924, 16354] dup_rows_per_dst=[4018, 9566, 2228, 154, 0, 0, 6, 602] combine_rows_per_src=[16380, 16206, 15992, 4888, 16378, 16376, 11924, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117474 payload_rows_remote=101134 dup_rows=13598 dup_rows_remote=5250 dedup_ratio=0.103745 router_tokens_per_expert=[1253, 2, 27, 8143, 4432, 2, 3, 0, 772, 7779, 8115, 8188, 3432, 7907, 149, 231, 8189, 0, 2, 3, 1, 3, 1, 0, 1, 56, 6268, 2, 2, 499, 3, 71] payload_rows_per_dst=[16334, 16340, 15748, 7438, 16384, 16384, 12538, 16308] dup_rows_per_dst=[2516, 8348, 1354, 186, 4, 2, 116, 1072] combine_rows_per_src=[16334, 16340, 15748, 7438, 16384, 16384, 12538, 16308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123784 payload_rows_remote=107608 dup_rows=7288 dup_rows_remote=6250 dedup_ratio=0.055603 router_tokens_per_expert=[613, 5, 140, 8123, 1566, 0, 4, 0, 536, 8071, 8136, 8184, 5080, 7733, 31, 44, 8185, 2, 2, 2, 8, 0, 0, 0, 8, 34, 8079, 8, 0, 285, 1, 656] payload_rows_per_dst=[16342, 15694, 16176, 10296, 16376, 16384, 16160, 16356] dup_rows_per_dst=[1420, 2912, 1038, 14, 6, 0, 98, 1800] combine_rows_per_src=[16342, 15694, 16176, 10296, 16376, 16384, 16160, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=97610 payload_rows_remote=81230 dup_rows=33462 dup_rows_remote=32450 dedup_ratio=0.255295 router_tokens_per_expert=[1743, 1, 79, 8080, 4403, 1, 0, 1, 816, 7891, 8189, 8191, 3096, 7748, 52, 14, 8186, 0, 1, 0, 0, 1, 0, 0, 0, 24, 6512, 0, 1, 251, 0, 255] payload_rows_per_dst=[3644, 8808, 3374, 16332, 16374, 16382, 16316, 16380] dup_rows_per_dst=[2, 2, 344, 19124, 22, 40, 12916, 1012] combine_rows_per_src=[3644, 8808, 3374, 16332, 16374, 16382, 16316, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108772 payload_rows_remote=92408 dup_rows=22300 dup_rows_remote=18682 dedup_ratio=0.170135 router_tokens_per_expert=[1284, 0, 9, 8086, 4658, 0, 0, 1, 1137, 7789, 8168, 8188, 2777, 7944, 42, 256, 8189, 0, 0, 0, 0, 0, 0, 0, 1, 18, 6767, 0, 1, 183, 1, 37] payload_rows_per_dst=[16364, 16308, 15966, 6028, 16378, 16376, 13536, 7816] dup_rows_per_dst=[3618, 8898, 9474, 122, 0, 0, 36, 152] combine_rows_per_src=[16364, 16308, 15966, 6028, 16378, 16376, 13536, 7816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119970 payload_rows_remote=103594 dup_rows=11102 dup_rows_remote=11102 dedup_ratio=0.084702 router_tokens_per_expert=[1319, 0, 6, 8183, 5067, 2, 0, 2, 515, 7943, 8170, 8185, 3133, 8013, 163, 32, 8192, 0, 0, 0, 2, 1, 0, 0, 0, 45, 6148, 2, 0, 383, 0, 30] payload_rows_per_dst=[16380, 10138, 15940, 16092, 16384, 16376, 12318, 16342] dup_rows_per_dst=[2636, 4, 976, 6590, 0, 0, 72, 824] combine_rows_per_src=[16380, 10138, 15940, 16092, 16384, 16376, 12318, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=12 export_slots=12 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 2, 4, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=810713088 dispatch_meta_bytes=442584 combine_payload_bytes=810713088 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=811155672 ingress_bytes=1163034624 link_bytes=1163034624 token_bytes=1621868760 weight_movement_bytes=352321536 total_nvlink_bytes=1974190296 weight_to_token_ratio=0.217232 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=27136 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.512508420e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=897925120 dispatch_meta_bytes=504120 combine_payload_bytes=897925120 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=898429240 ingress_bytes=1602568192 link_bytes=1602568192 token_bytes=1796354360 weight_movement_bytes=704643072 total_nvlink_bytes=2500997432 weight_to_token_ratio=0.392263 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=15744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.973712930e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120240 payload_rows_remote=103860 dup_rows=10832 dup_rows_remote=10832 dedup_ratio=0.082642 router_tokens_per_expert=[1252, 0, 5, 8184, 4325, 0, 1, 0, 629, 7644, 8182, 8184, 3112, 7846, 39, 33, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 6, 7296, 2, 1, 498, 0, 106] payload_rows_per_dst=[16380, 9878, 15384, 14890, 16380, 16368, 14592, 16368] dup_rows_per_dst=[2502, 346, 1162, 5598, 0, 0, 18, 1206] combine_rows_per_src=[16380, 9878, 15384, 14890, 16380, 16368, 14592, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=828489728 dispatch_meta_bytes=425536 combine_payload_bytes=828489728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=828915264 ingress_bytes=1180811264 link_bytes=1180811264 token_bytes=1657404992 weight_movement_bytes=352321536 total_nvlink_bytes=2009726528 weight_to_token_ratio=0.212574 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=17024 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.612050184e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=665436160 dispatch_meta_bytes=454720 combine_payload_bytes=665436160 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=665890880 ingress_bytes=1017757696 link_bytes=1017757696 token_bytes=1331327040 weight_movement_bytes=352321536 total_nvlink_bytes=1683648576 weight_to_token_ratio=0.264639 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=30592 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.699018265e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=881524736 dispatch_meta_bytes=455432 combine_payload_bytes=881524736 prefetch_bytes=0 reduce_bytes=2113929216 egress_bytes=881980168 ingress_bytes=2995453952 link_bytes=2995453952 token_bytes=1763504904 weight_movement_bytes=2113929216 total_nvlink_bytes=3877434120 weight_to_token_ratio=1.198709 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.677329177e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=757006336 dispatch_meta_bytes=444360 combine_payload_bytes=757006336 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=757450696 ingress_bytes=1461649408 link_bytes=1461649408 token_bytes=1514457032 weight_movement_bytes=704643072 total_nvlink_bytes=2219100104 weight_to_token_ratio=0.465278 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=27136 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.184626562e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=848642048 dispatch_meta_bytes=458784 combine_payload_bytes=848642048 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=849100832 ingress_bytes=1553285120 link_bytes=1553285120 token_bytes=1697742880 weight_movement_bytes=704643072 total_nvlink_bytes=2402385952 weight_to_token_ratio=0.415047 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.697748299e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=850821120 dispatch_meta_bytes=458768 combine_payload_bytes=850821120 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=851279888 ingress_bytes=1203142656 link_bytes=1203142656 token_bytes=1702101008 weight_movement_bytes=352321536 total_nvlink_bytes=2054422544 weight_to_token_ratio=0.206992 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.737096657e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114274 payload_rows_remote=97926 dup_rows=16798 dup_rows_remote=16516 dedup_ratio=0.128159 router_tokens_per_expert=[2710, 1, 8176, 1, 8186, 0, 167, 1, 2598, 8190, 0, 1, 8191, 1, 0, 0, 1, 1, 0, 1, 0, 0, 6446, 133, 1, 8167, 0, 147, 0, 4245, 0, 8171] payload_rows_per_dst=[5424, 16376, 13276, 16384, 16344, 13754, 16348, 16368] dup_rows_per_dst=[0, 332, 4442, 0, 4, 3264, 282, 8474] combine_rows_per_src=[5424, 16376, 13276, 16384, 16344, 13754, 16348, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123542 payload_rows_remote=107162 dup_rows=7530 dup_rows_remote=7526 dedup_ratio=0.057449 router_tokens_per_expert=[2044, 7, 8191, 0, 8189, 0, 122, 0, 1254, 8189, 0, 1, 8187, 0, 2, 3, 0, 0, 4, 0, 0, 0, 7650, 173, 0, 8167, 7, 178, 3, 4985, 0, 8180] payload_rows_per_dst=[16382, 16378, 16380, 16380, 16364, 15318, 16366, 9974] dup_rows_per_dst=[4102, 244, 2508, 4, 4, 328, 338, 2] combine_rows_per_src=[16382, 16378, 16380, 16380, 16364, 15318, 16366, 9974] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116448 payload_rows_remote=100076 dup_rows=14624 dup_rows_remote=13996 dedup_ratio=0.111572 router_tokens_per_expert=[3737, 0, 8174, 3, 8184, 1, 314, 1, 1724, 8188, 0, 3, 8189, 3, 1, 0, 2, 3, 0, 3, 0, 2, 6988, 100, 3, 8154, 0, 551, 0, 3019, 0, 8189] payload_rows_per_dst=[16384, 16372, 14458, 16384, 16384, 14060, 16368, 6038] dup_rows_per_dst=[10172, 628, 2644, 2, 10, 120, 1048, 0] combine_rows_per_src=[16384, 16372, 14458, 16384, 16384, 14060, 16368, 6038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=104260 payload_rows_remote=87878 dup_rows=26812 dup_rows_remote=19082 dedup_ratio=0.204559 router_tokens_per_expert=[2047, 27, 8190, 1, 8191, 0, 159, 0, 2494, 8191, 1, 0, 8187, 0, 0, 1, 0, 1, 2, 1, 1, 0, 7509, 88, 0, 8106, 0, 298, 0, 3865, 1, 8175] payload_rows_per_dst=[4144, 13176, 5216, 16376, 16356, 16366, 16244, 16382] dup_rows_per_dst=[6, 292, 326, 14, 2, 14660, 3782, 7730] combine_rows_per_src=[4144, 13176, 5216, 16376, 16356, 16366, 16244, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119196 payload_rows_remote=102820 dup_rows=11876 dup_rows_remote=7008 dedup_ratio=0.090607 router_tokens_per_expert=[2930, 6, 8170, 3, 8133, 1, 40, 0, 2436, 8185, 1, 0, 8184, 2, 10, 0, 8, 7, 1, 7, 0, 0, 6094, 42, 12, 8152, 14, 513, 8, 4391, 1, 8185] payload_rows_per_dst=[16370, 16294, 16376, 16382, 16384, 12220, 16372, 8798] dup_rows_per_dst=[5848, 54, 4868, 10, 32, 52, 1010, 2] combine_rows_per_src=[16370, 16294, 16376, 16382, 16384, 12220, 16372, 8798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=106638 payload_rows_remote=90254 dup_rows=24434 dup_rows_remote=2812 dedup_ratio=0.186417 router_tokens_per_expert=[2614, 6, 8191, 0, 8191, 0, 512, 0, 1093, 8192, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6395, 162, 0, 8153, 2, 712, 0, 4932, 0, 8191] payload_rows_per_dst=[16384, 16382, 9080, 16372, 16384, 12944, 9340, 9752] dup_rows_per_dst=[21622, 1030, 1040, 0, 112, 170, 460, 0] combine_rows_per_src=[16384, 16382, 9080, 16372, 16384, 12944, 9340, 9752] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=802209792 dispatch_meta_bytes=457768 combine_payload_bytes=802209792 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=802667560 ingress_bytes=1506852864 link_bytes=1506852864 token_bytes=1604877352 weight_movement_bytes=704643072 total_nvlink_bytes=2309520424 weight_to_token_ratio=0.439064 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.437747047e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119744 payload_rows_remote=105066 dup_rows=11328 dup_rows_remote=11060 dedup_ratio=0.086426 router_tokens_per_expert=[3456, 16, 8183, 0, 8182, 2, 269, 0, 1632, 8191, 0, 0, 8189, 0, 0, 1, 0, 0, 2, 0, 0, 0, 7317, 156, 0, 8149, 0, 154, 0, 3480, 0, 8157] payload_rows_per_dst=[6936, 16384, 16382, 16380, 16314, 14678, 16304, 16366] dup_rows_per_dst=[8, 522, 3264, 0, 4, 268, 302, 6960] combine_rows_per_src=[6936, 16384, 16382, 16380, 16314, 14678, 16304, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=877871104 dispatch_meta_bytes=458752 combine_payload_bytes=877871104 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=878329856 ingress_bytes=1230192640 link_bytes=1230192640 token_bytes=1756200960 weight_movement_bytes=352321536 total_nvlink_bytes=2108522496 weight_to_token_ratio=0.200616 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.888565276e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117800 payload_rows_remote=101428 dup_rows=13272 dup_rows_remote=13270 dedup_ratio=0.101257 router_tokens_per_expert=[3496, 25, 8190, 0, 8190, 3, 111, 0, 2214, 8192, 0, 0, 8191, 0, 1, 0, 1, 1, 0, 0, 0, 0, 6287, 148, 1, 8154, 1, 98, 0, 4047, 0, 8185] payload_rows_per_dst=[7030, 16382, 16384, 16384, 16372, 12732, 16312, 16204] dup_rows_per_dst=[260, 226, 4428, 0, 2, 138, 196, 8022] combine_rows_per_src=[7030, 16382, 16384, 16384, 16372, 12732, 16312, 16204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=819822592 dispatch_meta_bytes=456288 combine_payload_bytes=819822592 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=820278880 ingress_bytes=1524465664 link_bytes=1524465664 token_bytes=1640101472 weight_movement_bytes=704643072 total_nvlink_bytes=2344744544 weight_to_token_ratio=0.429634 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.536371375e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=719896576 dispatch_meta_bytes=427840 combine_payload_bytes=719896576 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=720324416 ingress_bytes=1776861184 link_bytes=1776861184 token_bytes=1440220992 weight_movement_bytes=1056964608 total_nvlink_bytes=2497185600 weight_to_token_ratio=0.733891 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.949680932e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=842301440 dispatch_meta_bytes=439312 combine_payload_bytes=842301440 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=842740752 ingress_bytes=1899266048 link_bytes=1899266048 token_bytes=1685042192 weight_movement_bytes=1056964608 total_nvlink_bytes=2742006800 weight_to_token_ratio=0.627263 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=24064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.063509708e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=739360768 dispatch_meta_bytes=372264 combine_payload_bytes=739360768 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=739733032 ingress_bytes=1444003840 link_bytes=1444003840 token_bytes=1479093800 weight_movement_bytes=704643072 total_nvlink_bytes=2183736872 weight_to_token_ratio=0.476402 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=16512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.085818747e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=860700672 dispatch_meta_bytes=464504 combine_payload_bytes=860700672 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=861165176 ingress_bytes=1213022208 link_bytes=1213022208 token_bytes=1721865848 weight_movement_bytes=352321536 total_nvlink_bytes=2074187384 weight_to_token_ratio=0.204616 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=28544 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.792418024e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=830898176 dispatch_meta_bytes=458792 combine_payload_bytes=830898176 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=831356968 ingress_bytes=1535541248 link_bytes=1535541248 token_bytes=1662255144 weight_movement_bytes=704643072 total_nvlink_bytes=2366898216 weight_to_token_ratio=0.423908 gemm_rows_defined=66816 gemm_rows_charged=67584 gemm_tail_pad_rows=768 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.011364 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.598390022e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125592 payload_rows_remote=109210 dup_rows=5480 dup_rows_remote=2116 dedup_ratio=0.041809 router_tokens_per_expert=[2, 0, 0, 1, 0, 8175, 0, 646, 6161, 3, 1, 0, 1, 0, 1, 0, 8170, 8188, 1, 8191, 74, 1, 0, 7843, 7, 2, 8191, 1673, 0, 1, 120, 8083] payload_rows_per_dst=[16384, 15840, 12326, 16342, 16378, 15730, 16382, 16210] dup_rows_per_dst=[4, 1258, 4, 2, 0, 650, 3364, 198] combine_rows_per_src=[16384, 15840, 12326, 16342, 16378, 15730, 16382, 16210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124960 payload_rows_remote=108618 dup_rows=6112 dup_rows_remote=6112 dedup_ratio=0.046631 router_tokens_per_expert=[0, 0, 0, 0, 0, 8136, 3, 625, 5787, 13, 0, 0, 0, 0, 0, 0, 8159, 8183, 0, 8172, 164, 0, 0, 7772, 4, 22, 8186, 2102, 0, 0, 176, 8032] payload_rows_per_dst=[16320, 16278, 11598, 16342, 16366, 15570, 16380, 16106] dup_rows_per_dst=[0, 1250, 2, 0, 0, 302, 4248, 310] combine_rows_per_src=[16320, 16278, 11598, 16342, 16366, 15570, 16380, 16106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=894648320 dispatch_meta_bytes=445304 combine_payload_bytes=894648320 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=895093624 ingress_bytes=1599291392 link_bytes=1599291392 token_bytes=1789741944 weight_movement_bytes=704643072 total_nvlink_bytes=2494385016 weight_to_token_ratio=0.393712 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020833 gemm_active_groups=12 gemm_peak_group_rows=23424 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.955364218e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117318 payload_rows_remote=101130 dup_rows=13754 dup_rows_remote=11888 dedup_ratio=0.104935 router_tokens_per_expert=[6, 0, 1, 3, 0, 8088, 2, 937, 5664, 14, 3, 0, 3, 0, 4, 0, 8151, 8183, 3, 8183, 102, 3, 0, 7638, 18, 7, 8189, 2069, 0, 3, 181, 8081] payload_rows_per_dst=[16374, 16188, 16384, 16308, 16372, 15310, 4158, 16224] dup_rows_per_dst=[14, 1866, 11356, 6, 0, 176, 30, 306] combine_rows_per_src=[16374, 16188, 16384, 16308, 16372, 15310, 4158, 16224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114338 payload_rows_remote=98120 dup_rows=16734 dup_rows_remote=16232 dedup_ratio=0.127670 router_tokens_per_expert=[1, 0, 0, 1, 0, 8139, 1, 1040, 6652, 3, 1, 0, 0, 1, 0, 0, 8187, 8138, 1, 8181, 140, 0, 0, 7640, 0, 3, 8166, 1027, 0, 1, 122, 8091] payload_rows_per_dst=[16366, 4040, 13312, 16374, 16278, 15410, 16340, 16218] dup_rows_per_dst=[2, 44, 0, 0, 0, 3106, 13080, 502] combine_rows_per_src=[16366, 4040, 13312, 16374, 16278, 15410, 16340, 16218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122328 payload_rows_remote=108346 dup_rows=8744 dup_rows_remote=5008 dedup_ratio=0.066711 router_tokens_per_expert=[20, 8, 0, 6, 0, 8178, 0, 1320, 6071, 40, 0, 4, 7, 0, 3, 1, 8180, 8071, 7, 8183, 364, 6, 0, 7337, 3, 2, 8168, 1359, 0, 6, 105, 8087] payload_rows_per_dst=[16376, 16366, 13982, 16384, 16156, 14908, 11964, 16192] dup_rows_per_dst=[48, 2630, 3736, 8, 0, 506, 1612, 204] combine_rows_per_src=[16376, 16366, 13982, 16384, 16156, 14908, 11964, 16192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114664 payload_rows_remote=98286 dup_rows=16408 dup_rows_remote=16310 dedup_ratio=0.125183 router_tokens_per_expert=[0, 0, 0, 0, 0, 8154, 0, 1640, 5523, 5, 0, 0, 0, 0, 0, 0, 8190, 8114, 0, 8176, 393, 0, 0, 7602, 19, 6, 8184, 1323, 0, 0, 137, 8070] payload_rows_per_dst=[16378, 16374, 16382, 16374, 16198, 15376, 2686, 14896] dup_rows_per_dst=[98, 3676, 11042, 198, 574, 614, 10, 196] combine_rows_per_src=[16378, 16374, 16382, 16374, 16198, 15376, 2686, 14896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125064 payload_rows_remote=109014 dup_rows=6008 dup_rows_remote=5654 dedup_ratio=0.045837 router_tokens_per_expert=[0, 0, 0, 0, 0, 8170, 0, 602, 5528, 5, 0, 0, 0, 0, 0, 0, 8186, 8188, 0, 8154, 178, 0, 0, 8024, 19, 8, 8186, 2109, 0, 0, 185, 7994] payload_rows_per_dst=[16324, 16340, 11060, 16356, 16376, 16050, 16384, 16174] dup_rows_per_dst=[0, 1204, 6, 0, 0, 354, 4260, 184] combine_rows_per_src=[16324, 16340, 11060, 16356, 16376, 16050, 16384, 16174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=889798656 dispatch_meta_bytes=458920 combine_payload_bytes=889798656 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=890257576 ingress_bytes=1594441728 link_bytes=1594441728 token_bytes=1780056232 weight_movement_bytes=704643072 total_nvlink_bytes=2484699304 weight_to_token_ratio=0.395854 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.928208124e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123838 payload_rows_remote=107464 dup_rows=7234 dup_rows_remote=7234 dedup_ratio=0.055191 router_tokens_per_expert=[0, 0, 0, 0, 0, 8169, 0, 690, 4935, 6, 0, 0, 0, 0, 0, 0, 8191, 8187, 0, 8190, 529, 0, 0, 7976, 0, 2, 8135, 1950, 0, 0, 510, 8066] payload_rows_per_dst=[16384, 16350, 9878, 16378, 16374, 15988, 16278, 16208] dup_rows_per_dst=[0, 1368, 4, 0, 0, 1022, 3896, 944] combine_rows_per_src=[16384, 16350, 9878, 16378, 16374, 15988, 16278, 16208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=828456960 dispatch_meta_bytes=452072 combine_payload_bytes=828456960 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=828909032 ingress_bytes=1885421568 link_bytes=1885421568 token_bytes=1657365992 weight_movement_bytes=1056964608 total_nvlink_bytes=2714330600 weight_to_token_ratio=0.637738 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.055757377e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=803799040 dispatch_meta_bytes=457408 combine_payload_bytes=803799040 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=804256448 ingress_bytes=1508442112 link_bytes=1508442112 token_bytes=1608055488 weight_movement_bytes=704643072 total_nvlink_bytes=2312698560 weight_to_token_ratio=0.438196 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.446646172e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=887570432 dispatch_meta_bytes=453416 combine_payload_bytes=887570432 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=888023848 ingress_bytes=1239891968 link_bytes=1239891968 token_bytes=1775594280 weight_movement_bytes=352321536 total_nvlink_bytes=2127915816 weight_to_token_ratio=0.198425 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=23424 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.942877464e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[4, 0, 0, 4, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=805158912 dispatch_meta_bytes=458384 combine_payload_bytes=805158912 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=805617296 ingress_bytes=1509801984 link_bytes=1509801984 token_bytes=1610776208 weight_movement_bytes=704643072 total_nvlink_bytes=2315419280 weight_to_token_ratio=0.437456 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.454260888e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=893042688 dispatch_meta_bytes=458672 combine_payload_bytes=893042688 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=893501360 ingress_bytes=1245364224 link_bytes=1245364224 token_bytes=1786544048 weight_movement_bytes=352321536 total_nvlink_bytes=2138865584 weight_to_token_ratio=0.197208 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.973519814e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110220 payload_rows_remote=93840 dup_rows=20852 dup_rows_remote=20852 dedup_ratio=0.159088 router_tokens_per_expert=[1, 1, 1, 8181, 2001, 0, 8191, 1, 0, 0, 4771, 8191, 1, 1145, 1, 7101, 7954, 0, 3501, 0, 1, 2, 4, 0, 0, 0, 8190, 0, 23, 1, 1, 6272] payload_rows_per_dst=[16364, 4142, 9542, 14964, 16238, 16384, 16380, 16206] dup_rows_per_dst=[4, 86, 0, 1532, 6672, 12, 0, 12546] combine_rows_per_src=[16364, 4142, 9542, 14964, 16238, 16384, 16380, 16206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=880345088 dispatch_meta_bytes=458792 combine_payload_bytes=880345088 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=880803880 ingress_bytes=2641952768 link_bytes=2641952768 token_bytes=1761148968 weight_movement_bytes=1761607680 total_nvlink_bytes=3522756648 weight_to_token_ratio=1.000260 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.479383270e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118014 payload_rows_remote=103538 dup_rows=13058 dup_rows_remote=11880 dedup_ratio=0.099625 router_tokens_per_expert=[0, 0, 0, 8186, 1328, 0, 8192, 0, 0, 9, 6123, 8187, 0, 664, 0, 6887, 7882, 0, 3153, 2, 0, 0, 2, 2, 0, 5, 8191, 0, 8, 0, 0, 6715] payload_rows_per_dst=[16372, 16384, 14024, 14476, 10562, 16382, 16382, 13432] dup_rows_per_dst=[0, 2656, 5980, 1178, 3220, 0, 10, 14] combine_rows_per_src=[16372, 16384, 14024, 14476, 10562, 16382, 16382, 13432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=768737280 dispatch_meta_bytes=458768 combine_payload_bytes=768737280 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=769196048 ingress_bytes=1121058816 link_bytes=1121058816 token_bytes=1537933328 weight_movement_bytes=352321536 total_nvlink_bytes=1890254864 weight_to_token_ratio=0.229088 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.277461417e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118414 payload_rows_remote=102032 dup_rows=12658 dup_rows_remote=9090 dedup_ratio=0.096573 router_tokens_per_expert=[3, 3, 4, 8189, 1778, 6, 8188, 3, 7, 1, 4009, 8189, 3, 1560, 3, 7461, 7190, 1, 4431, 0, 3, 14, 18, 0, 0, 0, 8188, 1, 27, 3, 0, 6253] payload_rows_per_dst=[16384, 16382, 15976, 15518, 8864, 16384, 16378, 12528] dup_rows_per_dst=[14, 3568, 6438, 2536, 0, 64, 0, 38] combine_rows_per_src=[16384, 16382, 15976, 15518, 8864, 16384, 16378, 12528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=109916 payload_rows_remote=93532 dup_rows=21156 dup_rows_remote=9460 dedup_ratio=0.161407 router_tokens_per_expert=[1, 1, 1, 8187, 2860, 17, 8191, 0, 3, 4, 4380, 8187, 1, 1166, 1, 7246, 7611, 0, 3631, 0, 1, 1, 5, 0, 0, 1, 8191, 0, 3, 1, 0, 5845] payload_rows_per_dst=[16376, 5754, 8774, 13852, 16018, 16376, 16382, 16384] dup_rows_per_dst=[4, 0, 0, 1738, 7694, 12, 12, 11696] combine_rows_per_src=[16376, 5754, 8774, 13852, 16018, 16376, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112638 payload_rows_remote=96526 dup_rows=18434 dup_rows_remote=7370 dedup_ratio=0.140640 router_tokens_per_expert=[7, 6, 8, 8180, 2463, 5, 8184, 8, 0, 7, 5849, 8183, 7, 1215, 7, 7567, 7732, 0, 3081, 0, 7, 95, 2, 0, 2, 1, 8183, 0, 101, 0, 2, 4634] payload_rows_per_dst=[16376, 16384, 16112, 15526, 6162, 16384, 16370, 9324] dup_rows_per_dst=[26, 4936, 11064, 2066, 0, 190, 2, 150] combine_rows_per_src=[16376, 16384, 16112, 15526, 6162, 16384, 16370, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=848183296 dispatch_meta_bytes=461672 combine_payload_bytes=848183296 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=848644968 ingress_bytes=1905147904 link_bytes=1905147904 token_bytes=1696828264 weight_movement_bytes=1056964608 total_nvlink_bytes=2753792872 weight_to_token_ratio=0.622906 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=27776 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.066803301e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119214 payload_rows_remote=102836 dup_rows=11858 dup_rows_remote=11854 dedup_ratio=0.090469 router_tokens_per_expert=[0, 0, 0, 8172, 795, 0, 8192, 0, 0, 19, 5149, 8187, 0, 1367, 0, 7297, 7017, 0, 4882, 0, 0, 0, 3, 1, 0, 3, 8188, 0, 44, 0, 0, 6220] payload_rows_per_dst=[16344, 15994, 10332, 15320, 15890, 16378, 16378, 12578] dup_rows_per_dst=[0, 1518, 4, 2008, 7908, 4, 4, 412] combine_rows_per_src=[16344, 15994, 10332, 15320, 15890, 16378, 16378, 12578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115696 payload_rows_remote=99364 dup_rows=15376 dup_rows_remote=15270 dedup_ratio=0.117310 router_tokens_per_expert=[0, 0, 0, 8166, 1195, 4, 8192, 0, 0, 4, 4280, 8186, 0, 1604, 0, 7009, 7142, 0, 4755, 0, 0, 0, 11, 1, 0, 3, 8191, 0, 9, 0, 0, 6784] payload_rows_per_dst=[16332, 16384, 15962, 11432, 9510, 16124, 16384, 13568] dup_rows_per_dst=[106, 6366, 7108, 1772, 0, 2, 4, 18] combine_rows_per_src=[16332, 16384, 15962, 11432, 9510, 16124, 16384, 13568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=835846144 dispatch_meta_bytes=444488 combine_payload_bytes=835846144 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=836290632 ingress_bytes=1540489216 link_bytes=1540489216 token_bytes=1672136776 weight_movement_bytes=704643072 total_nvlink_bytes=2376779848 weight_to_token_ratio=0.421403 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=24704 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.626096578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=118702 payload_rows_remote=102790 dup_rows=12370 dup_rows_remote=5948 dedup_ratio=0.094376 router_tokens_per_expert=[0, 0, 0, 8185, 2065, 3, 8191, 0, 0, 13, 4865, 8184, 0, 1182, 0, 7395, 7602, 0, 3565, 0, 0, 0, 3, 1, 0, 1, 8192, 0, 52, 0, 0, 6037] payload_rows_per_dst=[16370, 16384, 9756, 15414, 15912, 16374, 16384, 12108] dup_rows_per_dst=[0, 4134, 0, 1740, 6422, 2, 2, 70] combine_rows_per_src=[16370, 16384, 9756, 15414, 15912, 16374, 16384, 12108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=766214144 dispatch_meta_bytes=411968 combine_payload_bytes=766214144 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=766626112 ingress_bytes=1118535680 link_bytes=1118535680 token_bytes=1532840256 weight_movement_bytes=352321536 total_nvlink_bytes=1885161792 weight_to_token_ratio=0.229849 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.263332908e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=790740992 dispatch_meta_bytes=415584 combine_payload_bytes=790740992 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=791156576 ingress_bytes=1495384064 link_bytes=1495384064 token_bytes=1581897568 weight_movement_bytes=704643072 total_nvlink_bytes=2286540640 weight_to_token_ratio=0.445442 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=19840 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.373526554e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120328 payload_rows_remote=104100 dup_rows=10744 dup_rows_remote=8362 dedup_ratio=0.081970 router_tokens_per_expert=[170, 8191, 8191, 0, 8191, 8177, 3077, 1, 1, 8016, 0, 1, 8080, 3393, 2, 624, 1, 0, 1, 1, 8191, 0, 0, 0, 0, 0, 18, 1207, 0, 1, 0, 1] payload_rows_per_dst=[16382, 6156, 16034, 16378, 16384, 16382, 16228, 16384] dup_rows_per_dst=[340, 0, 2, 8014, 4, 0, 2382, 2] combine_rows_per_src=[16382, 6156, 16034, 16378, 16384, 16382, 16228, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=842432512 dispatch_meta_bytes=458760 combine_payload_bytes=842432512 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=842891272 ingress_bytes=1194754048 link_bytes=1194754048 token_bytes=1685323784 weight_movement_bytes=352321536 total_nvlink_bytes=2037645320 weight_to_token_ratio=0.209053 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.690123954e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=813989888 dispatch_meta_bytes=458536 combine_payload_bytes=813989888 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=814448424 ingress_bytes=1166311424 link_bytes=1166311424 token_bytes=1628438312 weight_movement_bytes=352321536 total_nvlink_bytes=1980759848 weight_to_token_ratio=0.216355 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.530857132e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=842055680 dispatch_meta_bytes=434952 combine_payload_bytes=842055680 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=842490632 ingress_bytes=1899020288 link_bytes=1899020288 token_bytes=1684546312 weight_movement_bytes=1056964608 total_nvlink_bytes=2741510920 weight_to_token_ratio=0.627448 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.063372092e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121602 payload_rows_remote=113972 dup_rows=9470 dup_rows_remote=9318 dedup_ratio=0.072250 router_tokens_per_expert=[172, 8192, 8186, 2, 8190, 8053, 3598, 0, 0, 7881, 0, 0, 8160, 2879, 18, 994, 0, 4, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1013, 0, 0, 2, 0] payload_rows_per_dst=[16376, 16356, 15762, 7630, 16384, 16384, 16326, 16384] dup_rows_per_dst=[344, 6946, 0, 152, 8, 0, 2020, 0] combine_rows_per_src=[16376, 16356, 15762, 7630, 16384, 16384, 16326, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=852787200 dispatch_meta_bytes=449848 combine_payload_bytes=852787200 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=853237048 ingress_bytes=1205108736 link_bytes=1205108736 token_bytes=1706024248 weight_movement_bytes=352321536 total_nvlink_bytes=2058345784 weight_to_token_ratio=0.206516 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.748105884e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120590 payload_rows_remote=104212 dup_rows=10482 dup_rows_remote=2648 dedup_ratio=0.079971 router_tokens_per_expert=[300, 8189, 8188, 0, 8188, 8167, 3936, 3, 3, 7850, 2, 3, 8086, 2604, 24, 743, 3, 0, 2, 3, 8188, 0, 1, 2, 0, 2, 9, 1036, 0, 4, 0, 0] payload_rows_per_dst=[16376, 16378, 15706, 8246, 16384, 16380, 14736, 16384] dup_rows_per_dst=[600, 7834, 10, 978, 10, 2, 1048, 0] combine_rows_per_src=[16376, 16378, 15706, 8246, 16384, 16380, 14736, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117330 payload_rows_remote=100950 dup_rows=13742 dup_rows_remote=13682 dedup_ratio=0.104843 router_tokens_per_expert=[126, 8191, 8189, 0, 8189, 8129, 2617, 1, 1, 7984, 0, 1, 8141, 3361, 16, 935, 1, 0, 1, 1, 8191, 0, 0, 1, 1, 0, 0, 1458, 0, 1, 0, 0] payload_rows_per_dst=[13494, 5990, 16052, 16350, 16384, 16382, 16298, 16380] dup_rows_per_dst=[234, 248, 1818, 8472, 4, 4, 2902, 60] combine_rows_per_src=[13494, 5990, 16052, 16350, 16384, 16382, 16298, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 2, 0, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=933658624 dispatch_meta_bytes=493160 combine_payload_bytes=933658624 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=934151784 ingress_bytes=1638301696 link_bytes=1638301696 token_bytes=1867810408 weight_movement_bytes=704643072 total_nvlink_bytes=2572453480 weight_to_token_ratio=0.377256 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.022727 gemm_active_groups=12 gemm_peak_group_rows=13696 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.173805637e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123932 payload_rows_remote=107708 dup_rows=7140 dup_rows_remote=7122 dedup_ratio=0.054474 router_tokens_per_expert=[207, 8182, 8184, 1, 8158, 8146, 1342, 6, 8, 8105, 1, 7, 7953, 3946, 10, 981, 6, 2, 12, 7, 8184, 0, 6, 5, 0, 3, 38, 2028, 0, 8, 0, 0] payload_rows_per_dst=[16370, 16334, 16224, 9726, 16384, 16380, 16182, 16332] dup_rows_per_dst=[414, 2654, 18, 148, 34, 10, 3862, 0] combine_rows_per_src=[16370, 16334, 16224, 9726, 16384, 16380, 16182, 16332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122276 payload_rows_remote=105892 dup_rows=8796 dup_rows_remote=8796 dedup_ratio=0.067108 router_tokens_per_expert=[215, 8192, 8189, 3, 8190, 8160, 4609, 0, 0, 7482, 0, 0, 8040, 2242, 15, 947, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 16, 1044, 0, 0, 0, 0] payload_rows_per_dst=[16384, 9218, 14964, 16382, 16384, 16384, 16180, 16380] dup_rows_per_dst=[430, 0, 0, 6346, 0, 0, 2020, 0] combine_rows_per_src=[16384, 9218, 14964, 16382, 16384, 16384, 16180, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116592 payload_rows_remote=100208 dup_rows=14480 dup_rows_remote=14232 dedup_ratio=0.110474 router_tokens_per_expert=[55, 8192, 8183, 0, 8191, 8149, 5020, 0, 0, 7963, 0, 0, 8043, 1181, 18, 1048, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 1298, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16302, 15926, 16306, 16312, 16380, 2598, 16384] dup_rows_per_dst=[248, 9958, 0, 4274, 0, 0, 0, 0] combine_rows_per_src=[16384, 16302, 15926, 16306, 16312, 16380, 2598, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=853704704 dispatch_meta_bytes=427440 combine_payload_bytes=853704704 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=854132144 ingress_bytes=2615312384 link_bytes=2615312384 token_bytes=1707836848 weight_movement_bytes=1761607680 total_nvlink_bytes=3469444528 weight_to_token_ratio=1.031485 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.464465767e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=826982400 dispatch_meta_bytes=458528 combine_payload_bytes=826982400 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=827440928 ingress_bytes=1531625472 link_bytes=1531625472 token_bytes=1654423328 weight_movement_bytes=704643072 total_nvlink_bytes=2359066400 weight_to_token_ratio=0.425915 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.576463311e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125800 payload_rows_remote=109418 dup_rows=5272 dup_rows_remote=5272 dedup_ratio=0.040222 router_tokens_per_expert=[1, 0, 0, 8191, 5558, 1, 0, 0, 1, 7, 1, 1, 8190, 581, 1, 8191, 8189, 0, 8186, 1, 1, 1, 8191, 2051, 0, 0, 0, 0, 1, 8191, 0, 0] payload_rows_per_dst=[16384, 11118, 16384, 16382, 16382, 16384, 16382, 16384] dup_rows_per_dst=[0, 0, 4, 1162, 2, 4104, 0, 0] combine_rows_per_src=[16384, 11118, 16384, 16382, 16382, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119658 payload_rows_remote=103274 dup_rows=11414 dup_rows_remote=11402 dedup_ratio=0.087082 router_tokens_per_expert=[157, 8192, 8191, 0, 8191, 8028, 4148, 0, 0, 7929, 0, 0, 8009, 2531, 36, 790, 0, 6, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 1135, 0, 0, 0, 0] payload_rows_per_dst=[16382, 12550, 15920, 9594, 16384, 16384, 16062, 16382] dup_rows_per_dst=[314, 4148, 2636, 2076, 12, 0, 2228, 0] combine_rows_per_src=[16382, 12550, 15920, 9594, 16384, 16384, 16062, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=882343936 dispatch_meta_bytes=459320 combine_payload_bytes=882343936 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=882803256 ingress_bytes=1234665472 link_bytes=1234665472 token_bytes=1765147192 weight_movement_bytes=352321536 total_nvlink_bytes=2117468728 weight_to_token_ratio=0.199599 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.913611268e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=867467264 dispatch_meta_bytes=458752 combine_payload_bytes=867467264 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=867926016 ingress_bytes=1219788800 link_bytes=1219788800 token_bytes=1735393280 weight_movement_bytes=352321536 total_nvlink_bytes=2087714816 weight_to_token_ratio=0.203021 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.830308115e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=820903936 dispatch_meta_bytes=457760 combine_payload_bytes=820903936 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=821361696 ingress_bytes=1877868544 link_bytes=1877868544 token_bytes=1642265632 weight_movement_bytes=1056964608 total_nvlink_bytes=2699230240 weight_to_token_ratio=0.643601 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.051527999e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=896352256 dispatch_meta_bytes=458760 combine_payload_bytes=896352256 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=896811016 ingress_bytes=1248673792 link_bytes=1248673792 token_bytes=1793163272 weight_movement_bytes=352321536 total_nvlink_bytes=2145484808 weight_to_token_ratio=0.196480 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.992052013e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126746 payload_rows_remote=110362 dup_rows=4326 dup_rows_remote=3584 dedup_ratio=0.033005 router_tokens_per_expert=[0, 0, 0, 8188, 6055, 0, 3, 0, 0, 0, 0, 0, 8192, 371, 0, 8192, 8176, 0, 8192, 16, 0, 0, 8184, 1778, 0, 0, 0, 0, 0, 8189, 0, 0] payload_rows_per_dst=[16376, 12110, 16370, 16384, 16376, 16368, 16384, 16378] dup_rows_per_dst=[0, 6, 0, 742, 22, 3556, 0, 0] combine_rows_per_src=[16376, 12110, 16370, 16384, 16376, 16368, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=846020608 dispatch_meta_bytes=458704 combine_payload_bytes=846020608 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=846479312 ingress_bytes=1198342144 link_bytes=1198342144 token_bytes=1692499920 weight_movement_bytes=352321536 total_nvlink_bytes=2044821456 weight_to_token_ratio=0.208166 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.710215794e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=120614 payload_rows_remote=104230 dup_rows=10458 dup_rows_remote=1224 dedup_ratio=0.079788 router_tokens_per_expert=[3, 0, 0, 8187, 4617, 3, 0, 1, 7, 10, 8, 3, 8183, 573, 4, 8188, 8189, 0, 8177, 2, 3, 4, 8188, 2984, 3, 0, 3, 0, 1, 8185, 10, 0] payload_rows_per_dst=[16380, 16384, 16382, 16374, 16362, 5974, 16380, 16378] dup_rows_per_dst=[0, 9234, 44, 1146, 4, 8, 8, 14] combine_rows_per_src=[16380, 16384, 16382, 16374, 16362, 5974, 16380, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119884 payload_rows_remote=103502 dup_rows=11188 dup_rows_remote=11168 dedup_ratio=0.085358 router_tokens_per_expert=[1, 0, 0, 8191, 5829, 1, 0, 0, 0, 4, 2, 1, 8190, 550, 1, 8191, 8191, 0, 8187, 5, 1, 1, 8188, 1809, 2, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 11660, 16148, 10166, 16382, 16378, 16384, 16382] dup_rows_per_dst=[0, 0, 6, 688, 224, 10248, 2, 20] combine_rows_per_src=[16384, 11660, 16148, 10166, 16382, 16378, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=904085504 dispatch_meta_bytes=455784 combine_payload_bytes=904085504 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=904541288 ingress_bytes=1961050112 link_bytes=1961050112 token_bytes=1808626792 weight_movement_bytes=1056964608 total_nvlink_bytes=2865591400 weight_to_token_ratio=0.584402 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.098106204e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126504 payload_rows_remote=110120 dup_rows=4568 dup_rows_remote=4372 dedup_ratio=0.034851 router_tokens_per_expert=[7, 1, 0, 8181, 5946, 8, 1, 10, 11, 79, 10, 7, 8135, 197, 8, 8176, 8181, 0, 8156, 2, 8, 9, 8181, 2004, 5, 0, 0, 27, 0, 8182, 4, 0] payload_rows_per_dst=[16378, 14534, 16384, 16286, 16312, 13874, 16372, 16364] dup_rows_per_dst=[0, 2328, 196, 394, 0, 1598, 44, 8] combine_rows_per_src=[16378, 14534, 16384, 16286, 16312, 13874, 16372, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125034 payload_rows_remote=108658 dup_rows=6038 dup_rows_remote=908 dedup_ratio=0.046066 router_tokens_per_expert=[0, 0, 0, 8191, 5183, 0, 1, 0, 0, 0, 2, 0, 8192, 450, 0, 8192, 8182, 0, 8192, 6, 0, 0, 8188, 2565, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16382, 10366, 16378, 16384, 16380, 16376, 16384, 16384] dup_rows_per_dst=[0, 2, 4, 900, 2, 5130, 0, 0] combine_rows_per_src=[16382, 10366, 16378, 16384, 16380, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116344 payload_rows_remote=99964 dup_rows=14728 dup_rows_remote=14702 dedup_ratio=0.112366 router_tokens_per_expert=[0, 0, 0, 8190, 7145, 0, 1, 0, 0, 0, 1, 0, 8192, 127, 0, 8192, 8190, 0, 8192, 5, 0, 0, 8188, 924, 0, 0, 0, 0, 0, 8189, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16238, 1848, 16348, 16378] dup_rows_per_dst=[26, 14284, 2, 416, 0, 0, 0, 0] combine_rows_per_src=[16380, 16384, 16384, 16384, 16238, 1848, 16348, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=853852160 dispatch_meta_bytes=421816 combine_payload_bytes=853852160 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=854273976 ingress_bytes=1206173696 link_bytes=1206173696 token_bytes=1708126136 weight_movement_bytes=352321536 total_nvlink_bytes=2060447672 weight_to_token_ratio=0.206262 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.754069216e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=847888384 dispatch_meta_bytes=458680 combine_payload_bytes=847888384 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=848347064 ingress_bytes=1200209920 link_bytes=1200209920 token_bytes=1696235448 weight_movement_bytes=352321536 total_nvlink_bytes=2048556984 weight_to_token_ratio=0.207708 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.720674560e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115598 payload_rows_remote=99216 dup_rows=15474 dup_rows_remote=10102 dedup_ratio=0.118057 router_tokens_per_expert=[1, 1, 8191, 7592, 1, 8180, 0, 1, 8188, 7779, 1, 0, 1, 0, 6527, 0, 8191, 0, 1, 8191, 2, 0, 0, 1, 1938, 0, 748, 0, 0, 0, 0, 1] payload_rows_per_dst=[3746, 16364, 15620, 14338, 16384, 16380, 16382, 16384] dup_rows_per_dst=[2, 134, 3642, 6322, 0, 2, 5372, 0] combine_rows_per_src=[3746, 16364, 15620, 14338, 16384, 16380, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127346 payload_rows_remote=110962 dup_rows=3726 dup_rows_remote=3724 dedup_ratio=0.028427 router_tokens_per_expert=[0, 0, 0, 8191, 6331, 0, 0, 0, 0, 0, 1, 0, 8192, 440, 0, 8192, 8192, 0, 8192, 1, 0, 0, 8192, 1421, 0, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16382, 12662, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 2, 880, 2, 2842, 0, 0] combine_rows_per_src=[16382, 12662, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=902103040 dispatch_meta_bytes=457968 combine_payload_bytes=902103040 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=902561008 ingress_bytes=1254424576 link_bytes=1254424576 token_bytes=1804664048 weight_movement_bytes=352321536 total_nvlink_bytes=2156985584 weight_to_token_ratio=0.195228 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.024254003e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=890126336 dispatch_meta_bytes=438264 combine_payload_bytes=890126336 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=890564600 ingress_bytes=1594769408 link_bytes=1594769408 token_bytes=1780690936 weight_movement_bytes=704643072 total_nvlink_bytes=2485334008 weight_to_token_ratio=0.395713 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.930042995e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=818905088 dispatch_meta_bytes=458664 combine_payload_bytes=818905088 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=819363752 ingress_bytes=1171226624 link_bytes=1171226624 token_bytes=1638268840 weight_movement_bytes=352321536 total_nvlink_bytes=1990590376 weight_to_token_ratio=0.215057 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.558380200e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=812777472 dispatch_meta_bytes=437272 combine_payload_bytes=812777472 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=813214744 ingress_bytes=1165099008 link_bytes=1165099008 token_bytes=1625992216 weight_movement_bytes=352321536 total_nvlink_bytes=1978313752 weight_to_token_ratio=0.216681 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.524068109e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127500 payload_rows_remote=113334 dup_rows=3572 dup_rows_remote=3572 dedup_ratio=0.027252 router_tokens_per_expert=[0, 0, 8177, 7694, 0, 8192, 0, 0, 8192, 8017, 0, 0, 0, 0, 7077, 6, 8192, 0, 0, 8192, 20, 0, 0, 0, 1259, 0, 512, 0, 0, 0, 6, 0] payload_rows_per_dst=[15388, 16384, 16034, 14166, 16384, 16384, 16376, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 40, 3520, 12] combine_rows_per_src=[15388, 16384, 16034, 14166, 16384, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=909000704 dispatch_meta_bytes=458744 combine_payload_bytes=909000704 prefetch_bytes=0 reduce_bytes=1409286144 egress_bytes=909459448 ingress_bytes=2318286848 link_bytes=2318286848 token_bytes=1818460152 weight_movement_bytes=1409286144 total_nvlink_bytes=3227746296 weight_to_token_ratio=0.774989 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.298143865e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117142 payload_rows_remote=100840 dup_rows=13930 dup_rows_remote=13930 dedup_ratio=0.106277 router_tokens_per_expert=[4, 3, 8189, 6799, 13, 8134, 1, 3, 8184, 7906, 4, 1, 3, 0, 6736, 0, 8188, 0, 3, 8187, 0, 2, 0, 4, 1635, 0, 1532, 2, 0, 0, 0, 3] payload_rows_per_dst=[16384, 16302, 15820, 13478, 16380, 16380, 6016, 16382] dup_rows_per_dst=[13606, 0, 2, 0, 0, 0, 322, 0] combine_rows_per_src=[16384, 16302, 15820, 13478, 16380, 16380, 6016, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=114354 payload_rows_remote=97970 dup_rows=16718 dup_rows_remote=16710 dedup_ratio=0.127548 router_tokens_per_expert=[1, 1, 8185, 7468, 1, 8191, 0, 1, 8191, 8070, 1, 1, 0, 0, 7443, 2, 8191, 0, 1, 8190, 44, 0, 0, 1, 938, 0, 609, 1, 0, 0, 4, 1] payload_rows_per_dst=[2, 16384, 16142, 16294, 16382, 16384, 16382, 16384] dup_rows_per_dst=[2, 2, 2, 13532, 0, 88, 3084, 8] combine_rows_per_src=[2, 16384, 16142, 16294, 16382, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=928432128 dispatch_meta_bytes=467624 combine_payload_bytes=928432128 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=928899752 ingress_bytes=1280753664 link_bytes=1280753664 token_bytes=1857331880 weight_movement_bytes=352321536 total_nvlink_bytes=2209653416 weight_to_token_ratio=0.189692 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.171685906e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=826081280 dispatch_meta_bytes=459080 combine_payload_bytes=826081280 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=826540360 ingress_bytes=1178402816 link_bytes=1178402816 token_bytes=1652621640 weight_movement_bytes=352321536 total_nvlink_bytes=2004943176 weight_to_token_ratio=0.213189 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.598563880e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119640 payload_rows_remote=104096 dup_rows=11432 dup_rows_remote=11422 dedup_ratio=0.087219 router_tokens_per_expert=[9, 9, 8180, 7117, 20, 8151, 0, 8, 8170, 7763, 9, 5, 0, 0, 5420, 4, 8182, 0, 8, 8178, 13, 11, 0, 8, 2978, 0, 1273, 10, 0, 0, 1, 9] payload_rows_per_dst=[15686, 16354, 15544, 10848, 16372, 16378, 12076, 16382] dup_rows_per_dst=[7260, 4, 10, 0, 0, 26, 4130, 2] combine_rows_per_src=[15686, 16354, 15544, 10848, 16372, 16378, 12076, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127368 payload_rows_remote=110984 dup_rows=3704 dup_rows_remote=3644 dedup_ratio=0.028259 router_tokens_per_expert=[0, 0, 8180, 7282, 0, 8188, 0, 0, 8192, 8035, 0, 0, 0, 0, 7407, 4, 8192, 0, 0, 8192, 30, 0, 0, 0, 1239, 0, 589, 0, 0, 0, 6, 0] payload_rows_per_dst=[14564, 16376, 16070, 14822, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 60, 3632, 12] combine_rows_per_src=[14564, 16376, 16070, 14822, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=802570240 dispatch_meta_bytes=458720 combine_payload_bytes=802570240 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=803028960 ingress_bytes=1154891776 link_bytes=1154891776 token_bytes=1605599200 weight_movement_bytes=352321536 total_nvlink_bytes=1957920736 weight_to_token_ratio=0.219433 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.466911870e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=110702 payload_rows_remote=94320 dup_rows=20370 dup_rows_remote=12134 dedup_ratio=0.155411 router_tokens_per_expert=[0, 8186, 3486, 0, 76, 0, 1, 1, 8191, 8189, 8191, 1, 1, 1, 0, 8189, 0, 1, 348, 6915, 7908, 1, 1, 0, 4119, 4, 0, 0, 0, 4, 0, 1722] payload_rows_per_dst=[6972, 16384, 7004, 16380, 15380, 15818, 16382, 16382] dup_rows_per_dst=[0, 154, 0, 44, 8482, 2, 8236, 3452] combine_rows_per_src=[6972, 16384, 7004, 16380, 15380, 15818, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117864 payload_rows_remote=101502 dup_rows=13208 dup_rows_remote=620 dedup_ratio=0.100769 router_tokens_per_expert=[0, 0, 8175, 6288, 0, 8192, 0, 0, 8192, 8058, 0, 0, 0, 0, 6760, 8, 8192, 0, 0, 8192, 49, 0, 0, 0, 2668, 0, 757, 0, 0, 0, 5, 0] payload_rows_per_dst=[16362, 16384, 16130, 13536, 16384, 16112, 6604, 16352] dup_rows_per_dst=[12588, 0, 356, 0, 18, 0, 246, 0] combine_rows_per_src=[16362, 16384, 16130, 13536, 16384, 16112, 6604, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126836 payload_rows_remote=110452 dup_rows=4236 dup_rows_remote=4236 dedup_ratio=0.032318 router_tokens_per_expert=[0, 0, 8183, 7741, 0, 8192, 0, 0, 8192, 7850, 0, 0, 0, 0, 6858, 10, 8192, 0, 0, 8192, 35, 0, 0, 0, 1713, 0, 377, 0, 0, 0, 1, 0] payload_rows_per_dst=[15482, 16384, 15700, 13736, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 70, 4164, 2] combine_rows_per_src=[15482, 16384, 15700, 13736, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=852754432 dispatch_meta_bytes=462072 combine_payload_bytes=852754432 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=853216504 ingress_bytes=1557397504 link_bytes=1557397504 token_bytes=1705970936 weight_movement_bytes=704643072 total_nvlink_bytes=2410614008 weight_to_token_ratio=0.413045 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.720775933e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=772669440 dispatch_meta_bytes=425816 combine_payload_bytes=772669440 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=773095256 ingress_bytes=1124990976 link_bytes=1124990976 token_bytes=1545764696 weight_movement_bytes=352321536 total_nvlink_bytes=1898086232 weight_to_token_ratio=0.227927 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.299479871e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=909180928 dispatch_meta_bytes=458512 combine_payload_bytes=909180928 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=909639440 ingress_bytes=1261502464 link_bytes=1261502464 token_bytes=1818820368 weight_movement_bytes=352321536 total_nvlink_bytes=2171141904 weight_to_token_ratio=0.193709 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.063887221e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116486 payload_rows_remote=100104 dup_rows=14586 dup_rows_remote=14586 dedup_ratio=0.111282 router_tokens_per_expert=[0, 8187, 4934, 0, 86, 0, 0, 0, 8190, 8192, 8192, 0, 0, 0, 0, 8191, 0, 0, 212, 5626, 7352, 0, 0, 0, 4205, 0, 0, 3, 0, 49, 0, 2117] payload_rows_per_dst=[16384, 16380, 16384, 16382, 11452, 14704, 8416, 16384] dup_rows_per_dst=[9858, 172, 0, 0, 224, 0, 0, 4332] combine_rows_per_src=[16384, 16380, 16384, 16382, 11452, 14704, 8416, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=831504384 dispatch_meta_bytes=408488 combine_payload_bytes=831504384 prefetch_bytes=352321536 reduce_bytes=1409286144 egress_bytes=831912872 ingress_bytes=2593112064 link_bytes=2593112064 token_bytes=1663417256 weight_movement_bytes=1761607680 total_nvlink_bytes=3425024936 weight_to_token_ratio=1.059029 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=22784 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.452034514e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115910 payload_rows_remote=99526 dup_rows=15162 dup_rows_remote=14690 dedup_ratio=0.115677 router_tokens_per_expert=[0, 8178, 1539, 1, 233, 0, 3, 4, 8188, 8185, 8188, 3, 4, 3, 0, 8173, 0, 3, 1235, 7186, 8101, 3, 4, 0, 4100, 11, 0, 0, 1, 3, 1, 2186] payload_rows_per_dst=[16360, 16384, 16382, 16354, 14764, 16210, 8222, 11234] dup_rows_per_dst=[3076, 472, 7044, 6, 2084, 6, 0, 2474] combine_rows_per_src=[16360, 16384, 16382, 16354, 14764, 16210, 8222, 11234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=904822784 dispatch_meta_bytes=458752 combine_payload_bytes=904822784 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=905281536 ingress_bytes=1609465856 link_bytes=1609465856 token_bytes=1810104320 weight_movement_bytes=704643072 total_nvlink_bytes=2514747392 weight_to_token_ratio=0.389283 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.012336969e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=105310 payload_rows_remote=88928 dup_rows=25762 dup_rows_remote=22860 dedup_ratio=0.196548 router_tokens_per_expert=[0, 8188, 4004, 0, 52, 0, 1, 1, 8191, 8189, 8191, 1, 1, 1, 0, 8180, 0, 1, 784, 5995, 7763, 1, 1, 0, 4539, 1, 0, 0, 0, 13, 0, 1438] payload_rows_per_dst=[8008, 14054, 2344, 16362, 16150, 15628, 16382, 16382] dup_rows_per_dst=[0, 94, 0, 2, 8990, 4700, 9074, 2902] combine_rows_per_src=[8008, 14054, 2344, 16362, 16150, 15628, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=820051968 dispatch_meta_bytes=458760 combine_payload_bytes=820051968 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=820510728 ingress_bytes=1172373504 link_bytes=1172373504 token_bytes=1640562696 weight_movement_bytes=352321536 total_nvlink_bytes=1992884232 weight_to_token_ratio=0.214757 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.564802250e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=815316992 dispatch_meta_bytes=456864 combine_payload_bytes=815316992 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=815773856 ingress_bytes=1167638528 link_bytes=1167638528 token_bytes=1631090848 weight_movement_bytes=352321536 total_nvlink_bytes=1983412384 weight_to_token_ratio=0.216004 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.538288360e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121118 payload_rows_remote=104738 dup_rows=9954 dup_rows_remote=9954 dedup_ratio=0.075943 router_tokens_per_expert=[0, 8135, 3579, 4, 173, 0, 8, 8, 8182, 8180, 8181, 10, 9, 5, 0, 8170, 0, 11, 454, 7106, 7875, 7, 10, 0, 4590, 48, 0, 0, 0, 6, 0, 785] payload_rows_per_dst=[16280, 16382, 16380, 16358, 14342, 15766, 9248, 16362] dup_rows_per_dst=[7156, 362, 0, 10, 800, 18, 28, 1580] combine_rows_per_src=[16280, 16382, 16380, 16358, 14342, 15766, 9248, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=728498176 dispatch_meta_bytes=447152 combine_payload_bytes=728498176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=728945328 ingress_bytes=1080819712 link_bytes=1080819712 token_bytes=1457443504 weight_movement_bytes=352321536 total_nvlink_bytes=1809765040 weight_to_token_ratio=0.241739 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=27392 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.052139231e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121592 payload_rows_remote=105396 dup_rows=9480 dup_rows_remote=9478 dedup_ratio=0.072327 router_tokens_per_expert=[0, 0, 2, 1, 136, 0, 0, 1, 8081, 107, 8028, 1, 1, 1, 1, 3883, 7215, 3236, 1, 0, 3, 0, 8191, 0, 1, 0, 0, 8098, 1, 2191, 8165, 8191] payload_rows_per_dst=[16384, 16284, 16262, 7768, 15942, 16382, 16196, 16374] dup_rows_per_dst=[4, 152, 10, 4, 4962, 6, 2, 4340] combine_rows_per_src=[16384, 16284, 16262, 7768, 15942, 16382, 16196, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117016 payload_rows_remote=101674 dup_rows=14056 dup_rows_remote=14056 dedup_ratio=0.107239 router_tokens_per_expert=[0, 8192, 3287, 0, 204, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8189, 0, 0, 290, 6438, 7671, 0, 0, 0, 5722, 0, 0, 0, 0, 48, 0, 919] payload_rows_per_dst=[6574, 16384, 16384, 16378, 13186, 15342, 16384, 16384] dup_rows_per_dst=[0, 408, 0, 0, 270, 0, 11444, 1934] combine_rows_per_src=[6574, 16384, 16384, 16378, 13186, 15342, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=104958 payload_rows_remote=88574 dup_rows=26114 dup_rows_remote=16582 dedup_ratio=0.199234 router_tokens_per_expert=[0, 8190, 4504, 0, 125, 0, 0, 0, 8192, 8191, 8192, 0, 0, 0, 0, 8187, 0, 0, 575, 5780, 7184, 0, 0, 0, 5265, 0, 0, 0, 0, 13, 0, 1138] payload_rows_per_dst=[16384, 16384, 16384, 16374, 12234, 14368, 10530, 2300] dup_rows_per_dst=[9532, 250, 15854, 0, 476, 0, 0, 2] combine_rows_per_src=[16384, 16384, 16384, 16374, 12234, 14368, 10530, 2300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 2, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=858013696 dispatch_meta_bytes=458768 combine_payload_bytes=858013696 prefetch_bytes=176160768 reduce_bytes=1585446912 egress_bytes=858472464 ingress_bytes=2619621376 link_bytes=2619621376 token_bytes=1716486160 weight_movement_bytes=1761607680 total_nvlink_bytes=3478093840 weight_to_token_ratio=1.026287 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=26752 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.466878622e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=863404032 dispatch_meta_bytes=459496 combine_payload_bytes=863404032 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=863863528 ingress_bytes=1215725568 link_bytes=1215725568 token_bytes=1727267560 weight_movement_bytes=352321536 total_nvlink_bytes=2079589096 weight_to_token_ratio=0.203976 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.807555712e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116864 payload_rows_remote=105002 dup_rows=14208 dup_rows_remote=13832 dedup_ratio=0.108398 router_tokens_per_expert=[0, 8192, 3958, 0, 212, 0, 0, 0, 8191, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 429, 5690, 7785, 0, 0, 0, 5209, 0, 0, 1, 0, 38, 0, 1255] payload_rows_per_dst=[11066, 16384, 16384, 16384, 11862, 15660, 12740, 16384] dup_rows_per_dst=[3422, 424, 528, 38, 376, 2850, 3986, 2584] combine_rows_per_src=[11066, 16384, 16384, 16384, 11862, 15660, 12740, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122660 payload_rows_remote=107144 dup_rows=8412 dup_rows_remote=3688 dedup_ratio=0.064178 router_tokens_per_expert=[0, 0, 4, 0, 3, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 2841, 7279, 4519, 0, 15, 0, 11, 8192, 0, 0, 0, 0, 8100, 0, 1845, 8153, 8192] payload_rows_per_dst=[16384, 16380, 16384, 15516, 9048, 16384, 16200, 16364] dup_rows_per_dst=[8, 6, 0, 4724, 20, 22, 0, 3632] combine_rows_per_src=[16384, 16380, 16384, 15516, 9048, 16384, 16200, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=832913408 dispatch_meta_bytes=462920 combine_payload_bytes=832913408 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=833376328 ingress_bytes=1185234944 link_bytes=1185234944 token_bytes=1666289736 weight_movement_bytes=352321536 total_nvlink_bytes=2018611272 weight_to_token_ratio=0.211441 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.636820945e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=725598208 dispatch_meta_bytes=420624 combine_payload_bytes=725598208 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=726018832 ingress_bytes=1782562816 link_bytes=1782562816 token_bytes=1451617040 weight_movement_bytes=1056964608 total_nvlink_bytes=2508581648 weight_to_token_ratio=0.728129 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=18304 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.981607691e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115972 payload_rows_remote=99730 dup_rows=15100 dup_rows_remote=14892 dedup_ratio=0.115204 router_tokens_per_expert=[4, 1, 22, 4, 186, 0, 0, 3, 8036, 276, 7750, 4, 3, 3, 4, 5243, 7339, 3044, 3, 0, 6, 0, 8185, 0, 3, 0, 0, 8055, 3, 991, 8182, 8186] payload_rows_per_dst=[16384, 16242, 16026, 16314, 10674, 16376, 16142, 7814] dup_rows_per_dst=[50, 208, 34, 8870, 2998, 28, 1446, 1466] combine_rows_per_src=[16384, 16242, 16026, 16314, 10674, 16376, 16142, 7814] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119336 payload_rows_remote=102952 dup_rows=11736 dup_rows_remote=7832 dedup_ratio=0.089539 router_tokens_per_expert=[0, 0, 35, 1, 104, 0, 0, 1, 8162, 4, 8106, 1, 1, 1, 1, 3017, 6730, 4909, 1, 19, 0, 4, 8191, 0, 0, 0, 0, 8039, 1, 1833, 8184, 8191] payload_rows_per_dst=[16144, 15988, 16222, 6036, 16114, 16370, 16078, 16384] dup_rows_per_dst=[70, 184, 362, 4, 7204, 8, 0, 3904] combine_rows_per_src=[16144, 15988, 16222, 6036, 16114, 16370, 16078, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=860176384 dispatch_meta_bytes=475336 combine_payload_bytes=860176384 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=860651720 ingress_bytes=1212497920 link_bytes=1212497920 token_bytes=1720828104 weight_movement_bytes=352321536 total_nvlink_bytes=2073149640 weight_to_token_ratio=0.204740 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022727 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.789482230e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=877723648 dispatch_meta_bytes=443328 combine_payload_bytes=877723648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=878166976 ingress_bytes=1230045184 link_bytes=1230045184 token_bytes=1755890624 weight_movement_bytes=352321536 total_nvlink_bytes=2108212160 weight_to_token_ratio=0.200651 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=17792 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.887739584e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 2, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=816988160 dispatch_meta_bytes=458488 combine_payload_bytes=816988160 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=817446648 ingress_bytes=1169309696 link_bytes=1169309696 token_bytes=1634434808 weight_movement_bytes=352321536 total_nvlink_bytes=1986756344 weight_to_token_ratio=0.215562 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.547646204e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126994 payload_rows_remote=110610 dup_rows=4078 dup_rows_remote=1732 dedup_ratio=0.031113 router_tokens_per_expert=[0, 1, 8187, 564, 1, 0, 0, 299, 0, 0, 1, 1, 0, 8191, 1, 8191, 8191, 1, 1, 8191, 0, 5937, 220, 0, 8191, 2, 1171, 1, 0, 1, 1, 8191] payload_rows_per_dst=[16376, 16384, 16384, 16384, 16384, 12314, 16384, 16384] dup_rows_per_dst=[1128, 598, 2, 0, 2, 0, 2346, 2] combine_rows_per_src=[16376, 16384, 16384, 16384, 16384, 12314, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=843382784 dispatch_meta_bytes=443136 combine_payload_bytes=843382784 prefetch_bytes=176160768 reduce_bytes=1409286144 egress_bytes=843825920 ingress_bytes=2428829696 link_bytes=2428829696 token_bytes=1687208704 weight_movement_bytes=1585446912 total_nvlink_bytes=3272655616 weight_to_token_ratio=0.939686 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.360043245e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121060 payload_rows_remote=105248 dup_rows=10012 dup_rows_remote=10002 dedup_ratio=0.076385 router_tokens_per_expert=[2, 0, 48, 10, 228, 0, 3, 8, 7932, 344, 7558, 9, 7, 5, 9, 5195, 5975, 4089, 8, 11, 3, 2, 8179, 0, 5, 0, 0, 8121, 8, 1418, 8178, 8181] payload_rows_per_dst=[16384, 16028, 15812, 15632, 8212, 16360, 16248, 16384] dup_rows_per_dst=[98, 314, 10, 6750, 4, 8, 4, 2824] combine_rows_per_src=[16384, 16028, 15812, 15632, 8212, 16360, 16248, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116284 payload_rows_remote=99900 dup_rows=14788 dup_rows_remote=14780 dedup_ratio=0.112823 router_tokens_per_expert=[0, 0, 4, 0, 37, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 3937, 7667, 3562, 0, 33, 0, 5, 8192, 0, 0, 0, 0, 8137, 0, 1253, 8134, 8192] payload_rows_per_dst=[16384, 16382, 16384, 15888, 16132, 16334, 16274, 2506] dup_rows_per_dst=[8, 74, 798, 7320, 6580, 8, 0, 0] combine_rows_per_src=[16384, 16382, 16384, 15888, 16132, 16334, 16274, 2506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121250 payload_rows_remote=104866 dup_rows=9822 dup_rows_remote=9812 dedup_ratio=0.074936 router_tokens_per_expert=[0, 0, 1, 0, 6, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 3429, 7747, 3638, 0, 28, 0, 5, 8192, 0, 0, 0, 0, 8145, 0, 1636, 8133, 8192] payload_rows_per_dst=[16384, 16384, 16384, 6858, 16196, 16384, 16290, 16370] dup_rows_per_dst=[2, 12, 0, 0, 6630, 10, 0, 3168] combine_rows_per_src=[16384, 16384, 16384, 6858, 16196, 16384, 16290, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=906117120 dispatch_meta_bytes=449368 combine_payload_bytes=906117120 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=906566488 ingress_bytes=1610760192 link_bytes=1610760192 token_bytes=1812683608 weight_movement_bytes=704643072 total_nvlink_bytes=2517326680 weight_to_token_ratio=0.388729 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.019584711e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=862191616 dispatch_meta_bytes=461000 combine_payload_bytes=862191616 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=862652616 ingress_bytes=1566834688 link_bytes=1566834688 token_bytes=1724844232 weight_movement_bytes=704643072 total_nvlink_bytes=2429487304 weight_to_token_ratio=0.408526 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.773620224e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129336 payload_rows_remote=112968 dup_rows=1736 dup_rows_remote=1736 dedup_ratio=0.013245 router_tokens_per_expert=[0, 0, 8192, 31, 0, 0, 0, 161, 0, 0, 0, 0, 6, 8178, 0, 8192, 8192, 0, 0, 8192, 0, 7339, 5, 0, 8192, 0, 674, 0, 0, 0, 0, 8182] payload_rows_per_dst=[16384, 16384, 16384, 16368, 16384, 14684, 16384, 16364] dup_rows_per_dst=[62, 322, 0, 0, 0, 4, 1348, 0] combine_rows_per_src=[16384, 16384, 16384, 16368, 16384, 14684, 16384, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121774 payload_rows_remote=112528 dup_rows=9298 dup_rows_remote=8188 dedup_ratio=0.070938 router_tokens_per_expert=[0, 0, 0, 0, 26, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 3230, 7675, 3958, 0, 9, 0, 4, 8192, 0, 0, 0, 0, 8096, 0, 1610, 8161, 8192] payload_rows_per_dst=[16384, 16382, 16384, 14452, 9246, 16384, 16192, 16350] dup_rows_per_dst=[0, 52, 0, 4936, 1110, 8, 0, 3192] combine_rows_per_src=[16384, 16382, 16384, 14452, 9246, 16384, 16192, 16350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=818380800 dispatch_meta_bytes=458720 combine_payload_bytes=818380800 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=818839520 ingress_bytes=1170702336 link_bytes=1170702336 token_bytes=1637220320 weight_movement_bytes=352321536 total_nvlink_bytes=1989541856 weight_to_token_ratio=0.215195 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.555444406e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123180 payload_rows_remote=106796 dup_rows=7892 dup_rows_remote=6336 dedup_ratio=0.060211 router_tokens_per_expert=[3, 4, 8154, 744, 3, 0, 2, 777, 3, 0, 3, 4, 0, 8188, 5, 8188, 8186, 4, 2, 8188, 0, 3995, 356, 0, 8188, 4, 2341, 3, 0, 0, 4, 8187] payload_rows_per_dst=[16320, 16384, 16384, 16384, 16380, 15348, 9598, 16382] dup_rows_per_dst=[1490, 1556, 12, 2, 4, 4114, 714, 0] combine_rows_per_src=[16320, 16384, 16384, 16384, 16380, 15348, 9598, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=859062272 dispatch_meta_bytes=458712 combine_payload_bytes=859062272 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=859520984 ingress_bytes=1387544576 link_bytes=1387544576 token_bytes=1718583256 weight_movement_bytes=528482304 total_nvlink_bytes=2247065560 weight_to_token_ratio=0.307510 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.769670436e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=108262 payload_rows_remote=91886 dup_rows=22810 dup_rows_remote=22756 dedup_ratio=0.174026 router_tokens_per_expert=[0, 1, 8184, 823, 1, 0, 1, 325, 6, 0, 2, 1, 0, 8174, 1, 8191, 8191, 1, 0, 8191, 0, 5888, 232, 0, 8191, 0, 944, 0, 0, 0, 1, 8187] payload_rows_per_dst=[7574, 9984, 16324, 13024, 16384, 12214, 16382, 16376] dup_rows_per_dst=[1206, 596, 16, 6066, 12, 26, 14834, 54] combine_rows_per_src=[7574, 9984, 16324, 13024, 16384, 12214, 16382, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=921829376 dispatch_meta_bytes=482864 combine_payload_bytes=921829376 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=922312240 ingress_bytes=1626472448 link_bytes=1626472448 token_bytes=1844141616 weight_movement_bytes=704643072 total_nvlink_bytes=2548784688 weight_to_token_ratio=0.382098 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=15872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.107566786e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=925433856 dispatch_meta_bytes=458816 combine_payload_bytes=925433856 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=925892672 ingress_bytes=1982398464 link_bytes=1982398464 token_bytes=1851326528 weight_movement_bytes=1056964608 total_nvlink_bytes=2908291136 weight_to_token_ratio=0.570923 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.110060390e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=874872832 dispatch_meta_bytes=452528 combine_payload_bytes=874872832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=875325360 ingress_bytes=1227194368 link_bytes=1227194368 token_bytes=1750198192 weight_movement_bytes=352321536 total_nvlink_bytes=2102519728 weight_to_token_ratio=0.201304 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.871776205e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130876 payload_rows_remote=114492 dup_rows=196 dup_rows_remote=126 dedup_ratio=0.001495 router_tokens_per_expert=[8182, 0, 0, 0, 0, 8187, 8191, 0, 1, 1, 8190, 0, 8191, 1, 8111, 8191, 31, 1, 1, 8189, 1, 0, 0, 1, 0, 1, 2, 33, 0, 1, 0, 29] payload_rows_per_dst=[16364, 16374, 16382, 16224, 16380, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 0, 64, 2, 70, 58] combine_rows_per_src=[16364, 16374, 16382, 16224, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=752730112 dispatch_meta_bytes=458568 combine_payload_bytes=752730112 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=753188680 ingress_bytes=1105051648 link_bytes=1105051648 token_bytes=1505918792 weight_movement_bytes=352321536 total_nvlink_bytes=1858240328 weight_to_token_ratio=0.233958 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.187827958e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125728 payload_rows_remote=109344 dup_rows=5344 dup_rows_remote=5314 dedup_ratio=0.040771 router_tokens_per_expert=[9, 9, 8126, 1097, 6, 0, 8, 975, 2, 1, 12, 10, 2, 8177, 13, 8179, 8177, 11, 0, 8182, 0, 4850, 735, 0, 8182, 45, 531, 6, 0, 0, 8, 8183] payload_rows_per_dst=[16280, 16378, 16384, 16380, 16376, 11170, 16378, 16382] dup_rows_per_dst=[2202, 1958, 30, 4, 0, 0, 1150, 0] combine_rows_per_src=[16280, 16378, 16384, 16380, 16376, 11170, 16378, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=112742 payload_rows_remote=96358 dup_rows=18330 dup_rows_remote=15744 dedup_ratio=0.139847 router_tokens_per_expert=[0, 0, 8192, 32, 0, 0, 0, 492, 0, 0, 0, 0, 2, 8141, 0, 8186, 8192, 0, 0, 8192, 0, 7359, 20, 0, 8191, 0, 351, 0, 0, 0, 0, 8186] payload_rows_per_dst=[16384, 16372, 16384, 13766, 16378, 16384, 702, 16372] dup_rows_per_dst=[2586, 984, 0, 4, 0, 14756, 0, 0] combine_rows_per_src=[16384, 16372, 16384, 13766, 16378, 16384, 702, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129546 payload_rows_remote=114654 dup_rows=1526 dup_rows_remote=1524 dedup_ratio=0.011642 router_tokens_per_expert=[0, 0, 8192, 96, 0, 0, 0, 93, 0, 0, 0, 0, 2, 8177, 0, 8191, 8192, 0, 0, 8192, 0, 7446, 1, 0, 8190, 0, 576, 0, 0, 0, 0, 8188] payload_rows_per_dst=[16384, 16384, 16384, 16358, 16384, 14892, 16384, 16376] dup_rows_per_dst=[192, 184, 0, 0, 0, 2, 1148, 0] combine_rows_per_src=[16384, 16384, 16384, 16358, 16384, 14892, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=937918464 dispatch_meta_bytes=458472 combine_payload_bytes=937918464 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938376936 ingress_bytes=1290240000 link_bytes=1290240000 token_bytes=1876295400 weight_movement_bytes=352321536 total_nvlink_bytes=2228616936 weight_to_token_ratio=0.187775 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224805427e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=789364736 dispatch_meta_bytes=448408 combine_payload_bytes=789364736 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=789813144 ingress_bytes=1494007808 link_bytes=1494007808 token_bytes=1579177880 weight_movement_bytes=704643072 total_nvlink_bytes=2283820952 weight_to_token_ratio=0.446209 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.365820095e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=895746048 dispatch_meta_bytes=458632 combine_payload_bytes=895746048 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=896204680 ingress_bytes=1248067584 link_bytes=1248067584 token_bytes=1791950728 weight_movement_bytes=352321536 total_nvlink_bytes=2144272264 weight_to_token_ratio=0.196613 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.988657501e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129786 payload_rows_remote=113402 dup_rows=1286 dup_rows_remote=1286 dedup_ratio=0.009811 router_tokens_per_expert=[0, 0, 8192, 60, 0, 0, 0, 111, 0, 0, 0, 0, 1, 8165, 0, 8191, 8192, 0, 0, 8192, 0, 7573, 9, 0, 8190, 0, 470, 0, 0, 0, 0, 8190] payload_rows_per_dst=[16384, 16382, 16384, 16332, 16384, 15156, 16384, 16380] dup_rows_per_dst=[120, 222, 0, 0, 0, 8, 936, 0] combine_rows_per_src=[16384, 16382, 16384, 16332, 16384, 15156, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130914 payload_rows_remote=114530 dup_rows=158 dup_rows_remote=158 dedup_ratio=0.001205 router_tokens_per_expert=[8191, 0, 0, 2, 0, 8192, 8191, 32, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16228, 16384, 16382, 16384] dup_rows_per_dst=[2, 64, 0, 0, 0, 0, 92, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16228, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130622 payload_rows_remote=114268 dup_rows=450 dup_rows_remote=450 dedup_ratio=0.003433 router_tokens_per_expert=[8151, 0, 1, 0, 0, 8177, 8187, 0, 4, 3, 8185, 0, 8188, 6, 8020, 8188, 70, 3, 4, 8189, 4, 0, 0, 4, 0, 4, 13, 37, 0, 3, 0, 95] payload_rows_per_dst=[16304, 16354, 16378, 16052, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 6, 0, 148, 8, 98, 190] combine_rows_per_src=[16304, 16354, 16378, 16052, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130426 payload_rows_remote=114044 dup_rows=646 dup_rows_remote=638 dedup_ratio=0.004929 router_tokens_per_expert=[8181, 0, 0, 0, 0, 8188, 8191, 4, 1, 1, 8191, 0, 8191, 2, 8117, 8191, 14, 1, 1, 8173, 1, 0, 0, 1, 0, 1, 3, 79, 0, 0, 0, 4] payload_rows_per_dst=[16362, 15950, 16378, 16238, 16348, 16384, 16384, 16382] dup_rows_per_dst=[0, 8, 2, 240, 222, 2, 164, 8] combine_rows_per_src=[16362, 15950, 16378, 16238, 16348, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=464712 combine_payload_bytes=939245568 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939710280 ingress_bytes=1291567104 link_bytes=1291567104 token_bytes=1878955848 weight_movement_bytes=352321536 total_nvlink_bytes=2231277384 weight_to_token_ratio=0.187509 gemm_rows_defined=65792 gemm_rows_charged=67584 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.026515 gemm_active_groups=8 gemm_peak_group_rows=25856 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232236656e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=928989184 dispatch_meta_bytes=458752 combine_payload_bytes=928989184 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=929447936 ingress_bytes=1985953792 link_bytes=1985953792 token_bytes=1858437120 weight_movement_bytes=1056964608 total_nvlink_bytes=2915401728 weight_to_token_ratio=0.568738 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.112051226e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=117460 payload_rows_remote=101120 dup_rows=13612 dup_rows_remote=13612 dedup_ratio=0.103851 router_tokens_per_expert=[1, 0, 6, 1, 46, 1, 0, 13, 8171, 6, 8191, 240, 2, 8186, 0, 1422, 8191, 1, 0, 1, 6517, 0, 0, 0, 8188, 0, 0, 8170, 2, 8178, 1, 1] payload_rows_per_dst=[16384, 16382, 16382, 2854, 16384, 16372, 16340, 16362] dup_rows_per_dst=[14, 114, 452, 0, 2, 13028, 0, 2] combine_rows_per_src=[16384, 16382, 16382, 2854, 16384, 16372, 16340, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938229760 dispatch_meta_bytes=458752 combine_payload_bytes=938229760 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=938688512 ingress_bytes=1995194368 link_bytes=1995194368 token_bytes=1876918272 weight_movement_bytes=1056964608 total_nvlink_bytes=2933882880 weight_to_token_ratio=0.563138 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117225563e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=936083456 dispatch_meta_bytes=458872 combine_payload_bytes=936083456 prefetch_bytes=352321536 reduce_bytes=1233125376 egress_bytes=936542328 ingress_bytes=2521530368 link_bytes=2521530368 token_bytes=1872625784 weight_movement_bytes=1585446912 total_nvlink_bytes=3458072696 weight_to_token_ratio=0.846644 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.411951752e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=934248448 dispatch_meta_bytes=458728 combine_payload_bytes=934248448 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934707176 ingress_bytes=1286569984 link_bytes=1286569984 token_bytes=1868955624 weight_movement_bytes=352321536 total_nvlink_bytes=2221277160 weight_to_token_ratio=0.188513 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.204254870e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129604 payload_rows_remote=113220 dup_rows=1468 dup_rows_remote=1210 dedup_ratio=0.011200 router_tokens_per_expert=[8192, 0, 0, 1, 0, 8192, 8192, 9, 0, 0, 8192, 0, 8192, 0, 8192, 8191, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16220, 16356, 15746, 15754] dup_rows_per_dst=[258, 326, 0, 884, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16376, 16384, 16220, 16356, 15746, 15754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130276 payload_rows_remote=113894 dup_rows=796 dup_rows_remote=782 dedup_ratio=0.006073 router_tokens_per_expert=[8112, 0, 10, 0, 0, 8155, 8181, 21, 10, 7, 8181, 0, 8176, 13, 7936, 8182, 99, 5, 9, 8148, 8, 0, 0, 9, 0, 11, 34, 23, 0, 1, 0, 205] payload_rows_per_dst=[16244, 16310, 16382, 15898, 16312, 16384, 16384, 16362] dup_rows_per_dst=[0, 42, 14, 0, 210, 16, 112, 402] combine_rows_per_src=[16244, 16310, 16382, 15898, 16312, 16384, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=828375040 dispatch_meta_bytes=458928 combine_payload_bytes=828375040 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=828833968 ingress_bytes=1709178880 link_bytes=1709178880 token_bytes=1657209008 weight_movement_bytes=880803840 total_nvlink_bytes=2538012848 weight_to_token_ratio=0.531498 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.570688282e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130972 payload_rows_remote=114588 dup_rows=100 dup_rows_remote=100 dedup_ratio=0.000763 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 21, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16284, 16384, 16384, 16384] dup_rows_per_dst=[0, 42, 0, 0, 0, 0, 58, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16284, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=927498240 dispatch_meta_bytes=457720 combine_payload_bytes=927498240 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=927955960 ingress_bytes=1279819776 link_bytes=1279819776 token_bytes=1855454200 weight_movement_bytes=352321536 total_nvlink_bytes=2207775736 weight_to_token_ratio=0.189884 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.166456523e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=933019648 dispatch_meta_bytes=458704 combine_payload_bytes=933019648 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=933478352 ingress_bytes=1461501952 link_bytes=1461501952 token_bytes=1866498000 weight_movement_bytes=528482304 total_nvlink_bytes=2394980304 weight_to_token_ratio=0.283141 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.183800870e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130958 payload_rows_remote=114686 dup_rows=114 dup_rows_remote=114 dedup_ratio=0.000870 router_tokens_per_expert=[8192, 0, 0, 1, 0, 8192, 8191, 33, 0, 0, 8192, 0, 8192, 0, 8192, 8191, 0, 0, 0, 8136, 0, 0, 0, 0, 1, 0, 0, 23, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16272, 16384, 16384, 16382] dup_rows_per_dst=[2, 66, 0, 0, 0, 0, 46, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16272, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124416 payload_rows_remote=108032 dup_rows=6656 dup_rows_remote=56 dedup_ratio=0.050781 router_tokens_per_expert=[0, 0, 0, 0, 4, 1, 2, 0, 8184, 0, 8192, 3, 0, 8191, 0, 3301, 8182, 0, 0, 0, 5146, 0, 15, 4, 8192, 0, 0, 7927, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16368, 16384, 16364, 10294, 15854, 16384] dup_rows_per_dst=[0, 14, 6, 6600, 0, 36, 0, 0] combine_rows_per_src=[16384, 16384, 16368, 16384, 16364, 10294, 15854, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128466 payload_rows_remote=112090 dup_rows=2606 dup_rows_remote=2312 dedup_ratio=0.019882 router_tokens_per_expert=[5, 2, 20, 4, 114, 0, 0, 46, 8124, 17, 8188, 417, 5, 8168, 0, 750, 8187, 3, 0, 2, 6978, 0, 0, 0, 8175, 0, 0, 8176, 12, 8136, 5, 2] payload_rows_per_dst=[16384, 16376, 16370, 16346, 16380, 13956, 16352, 16302] dup_rows_per_dst=[54, 294, 746, 1500, 4, 0, 0, 8] combine_rows_per_src=[16384, 16376, 16370, 16346, 16380, 13956, 16352, 16302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=115244 payload_rows_remote=98874 dup_rows=15828 dup_rows_remote=15528 dedup_ratio=0.120758 router_tokens_per_expert=[1, 1, 3, 1, 76, 0, 0, 8, 8149, 3, 8191, 261, 1, 8184, 0, 1648, 8186, 0, 1, 0, 6277, 0, 10, 0, 8190, 4, 0, 8155, 2, 8183, 1, 0] payload_rows_per_dst=[16166, 15312, 13278, 5068, 16372, 16368, 16310, 16370] dup_rows_per_dst=[10, 166, 466, 834, 26, 12574, 1452, 300] combine_rows_per_src=[16166, 15312, 13278, 5068, 16372, 16368, 16310, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938704896 dispatch_meta_bytes=458752 combine_payload_bytes=938704896 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939163648 ingress_bytes=1291026432 link_bytes=1291026432 token_bytes=1877868544 weight_movement_bytes=352321536 total_nvlink_bytes=2230190080 weight_to_token_ratio=0.187618 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229209118e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127582 payload_rows_remote=111198 dup_rows=3490 dup_rows_remote=3490 dedup_ratio=0.026627 router_tokens_per_expert=[6677, 8176, 7970, 1, 1, 0, 62, 1, 551, 8191, 0, 0, 1, 0, 0, 1, 8186, 8191, 0, 1, 1, 8189, 0, 2, 0, 0, 1, 8191, 1129, 3, 10, 0] payload_rows_per_dst=[13356, 16384, 16382, 16354, 16374, 16384, 16384, 15964] dup_rows_per_dst=[0, 126, 1102, 2, 0, 0, 0, 2260] combine_rows_per_src=[13356, 16384, 16382, 16354, 16374, 16384, 16384, 15964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=459200 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939966912 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879474624 weight_movement_bytes=352321536 total_nvlink_bytes=2231796160 weight_to_token_ratio=0.187457 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=884998144 dispatch_meta_bytes=432352 combine_payload_bytes=884998144 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=885430496 ingress_bytes=1589641216 link_bytes=1589641216 token_bytes=1770428640 weight_movement_bytes=704643072 total_nvlink_bytes=2475071712 weight_to_token_ratio=0.398007 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.901327261e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=918241280 dispatch_meta_bytes=457608 combine_payload_bytes=918241280 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=918698888 ingress_bytes=1270562816 link_bytes=1270562816 token_bytes=1836940168 weight_movement_bytes=352321536 total_nvlink_bytes=2189261704 weight_to_token_ratio=0.191798 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.018939 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.114621411e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=809975808 dispatch_meta_bytes=457608 combine_payload_bytes=809975808 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=810433416 ingress_bytes=1162297344 link_bytes=1162297344 token_bytes=1620409224 weight_movement_bytes=352321536 total_nvlink_bytes=1972730760 weight_to_token_ratio=0.217428 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.017045 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.508379960e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124460 payload_rows_remote=108076 dup_rows=6612 dup_rows_remote=6612 dedup_ratio=0.050446 router_tokens_per_expert=[0, 0, 0, 0, 2, 2, 2, 0, 8187, 0, 8192, 7, 0, 8191, 0, 3216, 8181, 0, 0, 0, 5071, 0, 13, 1, 8192, 0, 0, 8087, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16362, 10144, 16044, 16384] dup_rows_per_dst=[0, 12, 94, 6430, 50, 26, 0, 0] combine_rows_per_src=[16384, 16384, 16374, 16384, 16362, 10144, 16044, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124084 payload_rows_remote=107728 dup_rows=6988 dup_rows_remote=5552 dedup_ratio=0.053314 router_tokens_per_expert=[12, 7, 52, 10, 178, 0, 1, 101, 8038, 28, 8181, 830, 13, 8151, 0, 2462, 8181, 6, 0, 0, 4862, 0, 8, 1, 8144, 0, 0, 8174, 13, 8072, 11, 0] payload_rows_per_dst=[16384, 16370, 16356, 16328, 16374, 9738, 16348, 16186] dup_rows_per_dst=[140, 478, 1436, 4924, 0, 4, 0, 6] combine_rows_per_src=[16384, 16370, 16356, 16328, 16374, 9738, 16348, 16186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=910934016 dispatch_meta_bytes=458752 combine_payload_bytes=910934016 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=911392768 ingress_bytes=1087094784 link_bytes=1087094784 token_bytes=1822326784 weight_movement_bytes=176160768 total_nvlink_bytes=1998487552 weight_to_token_ratio=0.096668 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.087277015e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 2, 1, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124348 payload_rows_remote=110634 dup_rows=6724 dup_rows_remote=2816 dedup_ratio=0.051300 router_tokens_per_expert=[0, 0, 0, 0, 1, 1, 1, 0, 8185, 0, 8192, 0, 0, 8192, 0, 1666, 8187, 0, 0, 0, 6598, 0, 15, 1, 8192, 0, 0, 8113, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16370, 12512, 16374, 13714, 16226, 16384] dup_rows_per_dst=[0, 6, 0, 2810, 0, 3908, 0, 0] combine_rows_per_src=[16384, 16384, 16370, 12512, 16374, 13714, 16226, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=885358592 dispatch_meta_bytes=458752 combine_payload_bytes=885358592 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=885817344 ingress_bytes=1237680128 link_bytes=1237680128 token_bytes=1771175936 weight_movement_bytes=352321536 total_nvlink_bytes=2123497472 weight_to_token_ratio=0.198920 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.930492084e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=882507776 dispatch_meta_bytes=453120 combine_payload_bytes=882507776 prefetch_bytes=176160768 reduce_bytes=1409286144 egress_bytes=882960896 ingress_bytes=2467954688 link_bytes=2467954688 token_bytes=1765468672 weight_movement_bytes=1585446912 total_nvlink_bytes=3350915584 weight_to_token_ratio=0.898032 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.020833 gemm_active_groups=11 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.381951608e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125312 payload_rows_remote=108932 dup_rows=5760 dup_rows_remote=5760 dedup_ratio=0.043945 router_tokens_per_expert=[0, 0, 0, 0, 5, 0, 1, 0, 8179, 0, 8192, 8, 0, 8191, 0, 2858, 8190, 0, 0, 0, 5440, 0, 14, 1, 8192, 0, 0, 8073, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16362, 16384, 16380, 10888, 16146, 16384] dup_rows_per_dst=[0, 12, 12, 5714, 0, 22, 0, 0] combine_rows_per_src=[16384, 16384, 16362, 16384, 16380, 10888, 16146, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121240 payload_rows_remote=104856 dup_rows=9832 dup_rows_remote=9446 dedup_ratio=0.075012 router_tokens_per_expert=[6348, 8132, 7804, 4, 4, 0, 197, 4, 1243, 8188, 0, 0, 3, 0, 0, 4, 8172, 8180, 0, 5, 4, 8184, 0, 9, 0, 0, 3, 8187, 812, 16, 33, 0] payload_rows_per_dst=[13910, 16384, 16376, 16272, 16354, 16384, 16380, 9180] dup_rows_per_dst=[5680, 386, 2486, 6, 0, 10, 0, 1264] combine_rows_per_src=[13910, 16384, 16376, 16272, 16354, 16384, 16380, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123778 payload_rows_remote=107394 dup_rows=7294 dup_rows_remote=7294 dedup_ratio=0.055649 router_tokens_per_expert=[4545, 8192, 8192, 0, 0, 1, 0, 0, 231, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 3415, 0, 0, 0] payload_rows_per_dst=[9090, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 462, 0, 0, 0, 0, 6830] combine_rows_per_src=[9090, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=906313728 dispatch_meta_bytes=453800 combine_payload_bytes=906313728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=906767528 ingress_bytes=1258635264 link_bytes=1258635264 token_bytes=1813081256 weight_movement_bytes=352321536 total_nvlink_bytes=2165402792 weight_to_token_ratio=0.194322 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.047832098e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=126162 payload_rows_remote=109804 dup_rows=4910 dup_rows_remote=2380 dedup_ratio=0.037460 router_tokens_per_expert=[6598, 8178, 8173, 1, 1, 3, 22, 1, 335, 8191, 0, 0, 1, 0, 0, 1, 8184, 8190, 0, 1, 1, 8191, 0, 1, 0, 0, 1, 8191, 1263, 4, 4, 0] payload_rows_per_dst=[12752, 15188, 16382, 16358, 16370, 16370, 16384, 16358] dup_rows_per_dst=[0, 50, 670, 448, 1196, 2, 14, 2530] combine_rows_per_src=[12752, 15188, 16382, 16358, 16370, 16370, 16384, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127642 payload_rows_remote=111262 dup_rows=3430 dup_rows_remote=322 dedup_ratio=0.026169 router_tokens_per_expert=[0, 3, 1, 0, 6853, 1, 1, 8190, 1, 1, 1, 1, 155, 1, 0, 0, 8188, 0, 7933, 8191, 1, 0, 1, 8191, 1554, 0, 0, 0, 1, 8076, 0, 8191] payload_rows_per_dst=[16384, 13708, 16384, 16382, 15866, 16384, 16380, 16154] dup_rows_per_dst=[6, 2, 6, 306, 0, 2, 3108, 0] combine_rows_per_src=[16384, 13708, 16384, 16382, 15866, 16384, 16380, 16154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=892370944 dispatch_meta_bytes=458768 combine_payload_bytes=892370944 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=892829712 ingress_bytes=1244692480 link_bytes=1244692480 token_bytes=1785200656 weight_movement_bytes=352321536 total_nvlink_bytes=2137522192 weight_to_token_ratio=0.197357 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.969758328e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=899514368 dispatch_meta_bytes=448736 combine_payload_bytes=899514368 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=899963104 ingress_bytes=1251835904 link_bytes=1251835904 token_bytes=1799477472 weight_movement_bytes=352321536 total_nvlink_bytes=2151799008 weight_to_token_ratio=0.195791 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.009758520e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=858980352 dispatch_meta_bytes=457208 combine_payload_bytes=858980352 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=859437560 ingress_bytes=1211301888 link_bytes=1211301888 token_bytes=1718417912 weight_movement_bytes=352321536 total_nvlink_bytes=2070739448 weight_to_token_ratio=0.205027 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.782784951e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124062 payload_rows_remote=107698 dup_rows=7010 dup_rows_remote=3036 dedup_ratio=0.053482 router_tokens_per_expert=[5720, 8030, 7296, 12, 11, 7, 313, 10, 1988, 8181, 0, 0, 7, 0, 0, 11, 8146, 8157, 0, 12, 9, 8178, 0, 28, 0, 0, 7, 8180, 1152, 29, 51, 1] payload_rows_per_dst=[11464, 16384, 16364, 16082, 16316, 16384, 16374, 14694] dup_rows_per_dst=[0, 612, 3974, 14, 0, 46, 0, 2364] combine_rows_per_src=[11464, 16384, 16364, 16082, 16316, 16384, 16374, 14694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=879771648 dispatch_meta_bytes=458752 combine_payload_bytes=879771648 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=880230400 ingress_bytes=1584414720 link_bytes=1584414720 token_bytes=1760002048 weight_movement_bytes=704643072 total_nvlink_bytes=2464645120 weight_to_token_ratio=0.400365 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.872061065e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=106240 payload_rows_remote=89856 dup_rows=24832 dup_rows_remote=340 dedup_ratio=0.189453 router_tokens_per_expert=[6085, 8192, 8192, 0, 0, 4, 0, 0, 418, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 1685, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 5682, 16384, 15310, 16342, 16384, 3370] dup_rows_per_dst=[24492, 8, 332, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 5682, 16384, 15310, 16342, 16384, 3370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 4, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=911458304 dispatch_meta_bytes=446336 combine_payload_bytes=911458304 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=911904640 ingress_bytes=1263779840 link_bytes=1263779840 token_bytes=1823362944 weight_movement_bytes=352321536 total_nvlink_bytes=2175684480 weight_to_token_ratio=0.193226 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=22656 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.076639576e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=128446 payload_rows_remote=112062 dup_rows=2626 dup_rows_remote=2626 dedup_ratio=0.020035 router_tokens_per_expert=[6879, 8192, 8192, 0, 0, 6, 0, 0, 282, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 1025, 0, 0, 0] payload_rows_per_dst=[13758, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 12, 564, 0, 0, 0, 0, 2050] combine_rows_per_src=[13758, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=882262016 dispatch_meta_bytes=442936 combine_payload_bytes=882262016 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=882704952 ingress_bytes=1586905088 link_bytes=1586905088 token_bytes=1764966968 weight_movement_bytes=704643072 total_nvlink_bytes=2469610040 weight_to_token_ratio=0.399239 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.886006086e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=736100352 dispatch_meta_bytes=360784 combine_payload_bytes=736100352 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=736461136 ingress_bytes=2145386496 link_bytes=2145386496 token_bytes=1472561488 weight_movement_bytes=1409286144 total_nvlink_bytes=2881847632 weight_to_token_ratio=0.957030 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020833 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.201326885e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124872 payload_rows_remote=108488 dup_rows=6200 dup_rows_remote=6200 dedup_ratio=0.047302 router_tokens_per_expert=[5093, 8192, 8192, 0, 0, 0, 0, 0, 572, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8192, 2527, 0, 0, 1] payload_rows_per_dst=[10186, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 1144, 0, 0, 0, 0, 5056] combine_rows_per_src=[10186, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=113632 payload_rows_remote=97522 dup_rows=17440 dup_rows_remote=15012 dedup_ratio=0.133057 router_tokens_per_expert=[1, 0, 1, 0, 6518, 1, 1, 8190, 0, 0, 1, 1, 172, 1, 0, 0, 8191, 0, 7473, 8190, 1, 0, 0, 8191, 2355, 0, 0, 2, 1, 8054, 0, 8191] payload_rows_per_dst=[16310, 3146, 16336, 12706, 16260, 16384, 16380, 16110] dup_rows_per_dst=[2, 2, 2, 344, 9948, 0, 4714, 2428] combine_rows_per_src=[16310, 3146, 16336, 12706, 16260, 16384, 16380, 16110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=918011904 dispatch_meta_bytes=458752 combine_payload_bytes=918011904 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=918470656 ingress_bytes=1270333440 link_bytes=1270333440 token_bytes=1836482560 weight_movement_bytes=352321536 total_nvlink_bytes=2188804096 weight_to_token_ratio=0.191846 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.113337001e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=116940 payload_rows_remote=100556 dup_rows=14132 dup_rows_remote=1168 dedup_ratio=0.107819 router_tokens_per_expert=[2, 0, 4, 0, 6482, 4, 4, 8184, 3, 0, 4, 4, 575, 5, 0, 0, 8182, 0, 7914, 8188, 4, 0, 2, 8187, 1787, 0, 0, 4, 4, 7805, 0, 8188] payload_rows_per_dst=[16384, 16384, 16384, 16376, 15828, 16384, 3582, 15618] dup_rows_per_dst=[4, 12964, 14, 1148, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16376, 15828, 16384, 3582, 15618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=122250 payload_rows_remote=105866 dup_rows=8822 dup_rows_remote=8802 dedup_ratio=0.067307 router_tokens_per_expert=[0, 0, 0, 0, 4839, 0, 0, 8192, 0, 0, 0, 0, 10, 0, 0, 0, 8192, 0, 7135, 8191, 0, 0, 0, 8192, 4399, 0, 0, 2, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 9678, 16382, 16384, 14270, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 20, 0, 0, 8802, 0] combine_rows_per_src=[16384, 9678, 16382, 16384, 14270, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=888733696 dispatch_meta_bytes=458752 combine_payload_bytes=888733696 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=889192448 ingress_bytes=1769537536 link_bytes=1769537536 token_bytes=1777926144 weight_movement_bytes=880803840 total_nvlink_bytes=2658729984 weight_to_token_ratio=0.495411 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.908671560e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130230 payload_rows_remote=113846 dup_rows=842 dup_rows_remote=842 dedup_ratio=0.006424 router_tokens_per_expert=[8150, 7933, 8190, 8187, 2, 2, 0, 448, 1, 0, 8190, 11, 1, 8191, 0, 0, 0, 1, 0, 0, 0, 1, 8035, 0, 0, 0, 8191, 1, 1, 0, 0, 0] payload_rows_per_dst=[15866, 16384, 16382, 16384, 16382, 16072, 16384, 16376] dup_rows_per_dst=[0, 820, 22, 0, 0, 0, 0, 0] combine_rows_per_src=[15866, 16384, 16382, 16384, 16382, 16072, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=798900224 dispatch_meta_bytes=450136 combine_payload_bytes=798900224 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=799350360 ingress_bytes=1503543296 link_bytes=1503543296 token_bytes=1598250584 weight_movement_bytes=704643072 total_nvlink_bytes=2302893656 weight_to_token_ratio=0.440884 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.419214848e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=823754752 dispatch_meta_bytes=406896 combine_payload_bytes=823754752 prefetch_bytes=352321536 reduce_bytes=1056964608 egress_bytes=824161648 ingress_bytes=2233040896 link_bytes=2233040896 token_bytes=1647916400 weight_movement_bytes=1409286144 total_nvlink_bytes=3057202544 weight_to_token_ratio=0.855193 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=22656 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.250409690e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=121060 payload_rows_remote=104676 dup_rows=10012 dup_rows_remote=9984 dedup_ratio=0.076385 router_tokens_per_expert=[14, 23, 9, 0, 6930, 11, 8, 8172, 0, 1, 12, 12, 938, 7, 0, 0, 8154, 0, 7789, 8181, 12, 0, 11, 8179, 1280, 0, 0, 0, 10, 7603, 0, 8180] payload_rows_per_dst=[16384, 15938, 16384, 16368, 15578, 16384, 8798, 15226] dup_rows_per_dst=[68, 7478, 28, 1830, 0, 20, 588, 0] combine_rows_per_src=[16384, 15938, 16384, 16368, 15578, 16384, 8798, 15226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=119518 payload_rows_remote=103136 dup_rows=11554 dup_rows_remote=11554 dedup_ratio=0.088150 router_tokens_per_expert=[0, 0, 0, 0, 5754, 0, 0, 8192, 0, 0, 0, 0, 10, 0, 0, 0, 8192, 0, 6753, 8191, 0, 0, 0, 8192, 3867, 0, 0, 1, 0, 8192, 0, 8192] payload_rows_per_dst=[16382, 16384, 16384, 16384, 13506, 16360, 7734, 16384] dup_rows_per_dst=[0, 11532, 0, 20, 0, 0, 2, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 13506, 16360, 7734, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=867254272 dispatch_meta_bytes=458672 combine_payload_bytes=867254272 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=867712944 ingress_bytes=1219575808 link_bytes=1219575808 token_bytes=1734967216 weight_movement_bytes=352321536 total_nvlink_bytes=2087288752 weight_to_token_ratio=0.203071 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.829115449e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=932626432 dispatch_meta_bytes=458752 combine_payload_bytes=932626432 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=933085184 ingress_bytes=1108787200 link_bytes=1108787200 token_bytes=1865711616 weight_movement_bytes=176160768 total_nvlink_bytes=2041872384 weight_to_token_ratio=0.094420 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.024621 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.208745489e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=125962 payload_rows_remote=109578 dup_rows=5110 dup_rows_remote=5110 dedup_ratio=0.038986 router_tokens_per_expert=[0, 0, 0, 0, 6473, 0, 0, 8192, 0, 0, 0, 0, 13, 0, 0, 0, 8192, 0, 7358, 8190, 0, 0, 0, 8192, 2540, 0, 0, 2, 0, 8192, 0, 8192] payload_rows_per_dst=[16380, 12946, 16384, 16384, 14716, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 26, 0, 0, 5084, 0] combine_rows_per_src=[16380, 12946, 16384, 16384, 14716, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=857505792 dispatch_meta_bytes=458640 combine_payload_bytes=857505792 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=857964432 ingress_bytes=1209827328 link_bytes=1209827328 token_bytes=1715470224 weight_movement_bytes=352321536 total_nvlink_bytes=2067791760 weight_to_token_ratio=0.205379 gemm_rows_defined=66560 gemm_rows_charged=67584 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.015152 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.774528030e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=844890112 dispatch_meta_bytes=458760 combine_payload_bytes=844890112 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=845348872 ingress_bytes=1197211648 link_bytes=1197211648 token_bytes=1690238984 weight_movement_bytes=352321536 total_nvlink_bytes=2042560520 weight_to_token_ratio=0.208445 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.703885488e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27177730680 prefetch_bytes=5108662272 reduce_bytes=1761607680 weight_movement_bytes=6870269952 egress_bytes=13592478328 ingress_bytes=20455522304 link_bytes=20455522304 total_nvlink_bytes=34048000632 weight_to_token_ratio=0.252790 prefetch_share_of_ingress=0.249745 dedup_ratio=0.086701 prefetch_slots=58 unused_prefetch_slots=198 slot_utilization=0.226562 reduce_slots=10 remote_demand=29 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.021188 gemm_inner_pad_upper_bound=19939 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.158899147e-05 prefetch_bytes_per_gemm_flop=1.787898516e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=124314 payload_rows_remote=109214 dup_rows=6758 dup_rows_remote=6758 dedup_ratio=0.051559 router_tokens_per_expert=[0, 0, 0, 0, 5455, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 7550, 8192, 0, 0, 0, 8192, 3374, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 10910, 16384, 16384, 15100, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 10, 0, 0, 6748, 0] combine_rows_per_src=[16384, 10910, 16384, 16384, 15100, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=127872 payload_rows_remote=111496 dup_rows=3200 dup_rows_remote=3162 dedup_ratio=0.024414 router_tokens_per_expert=[8007, 7979, 8178, 8189, 1, 2, 0, 403, 1, 1, 8190, 35, 1, 8191, 0, 1, 0, 1, 0, 0, 1, 1, 8162, 0, 0, 0, 8191, 1, 0, 0, 0, 0] payload_rows_per_dst=[13286, 16384, 16372, 16384, 16360, 16326, 16384, 16376] dup_rows_per_dst=[0, 442, 72, 10, 102, 2534, 2, 38] combine_rows_per_src=[13286, 16384, 16372, 16384, 16360, 16326, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=897662976 dispatch_meta_bytes=458752 combine_payload_bytes=897662976 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=898121728 ingress_bytes=1249984512 link_bytes=1249984512 token_bytes=1795784704 weight_movement_bytes=352321536 total_nvlink_bytes=2148106240 weight_to_token_ratio=0.196194 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.999391498e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130034 payload_rows_remote=113650 dup_rows=1038 dup_rows_remote=24 dedup_ratio=0.007919 router_tokens_per_expert=[7912, 8145, 8187, 8171, 5, 8, 8, 766, 3, 0, 8178, 17, 4, 8186, 0, 0, 0, 4, 3, 0, 0, 5, 7738, 0, 0, 0, 8188, 4, 4, 0, 0, 0] payload_rows_per_dst=[16290, 16384, 16374, 16380, 16362, 15486, 16384, 16374] dup_rows_per_dst=[0, 1014, 22, 0, 2, 0, 0, 0] combine_rows_per_src=[16290, 16384, 16374, 16380, 16362, 15486, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=130370 payload_rows_remote=113986 dup_rows=702 dup_rows_remote=702 dedup_ratio=0.005356 router_tokens_per_expert=[8176, 7856, 8190, 8192, 0, 0, 0, 343, 0, 0, 8192, 8, 0, 8192, 0, 0, 0, 0, 0, 0, 12, 0, 8183, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[15712, 16376, 16384, 16384, 16382, 16366, 16384, 16382] dup_rows_per_dst=[0, 662, 16, 0, 0, 24, 0, 0] combine_rows_per_src=[15712, 16376, 16384, 16384, 16382, 16366, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=894681088 dispatch_meta_bytes=463888 combine_payload_bytes=894681088 prefetch_bytes=352321536 reduce_bytes=704643072 egress_bytes=895144976 ingress_bytes=1951645696 link_bytes=1951645696 token_bytes=1789826064 weight_movement_bytes=1056964608 total_nvlink_bytes=2846790672 weight_to_token_ratio=0.590540 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022727 gemm_active_groups=9 gemm_peak_group_rows=30592 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.092840124e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=913375232 dispatch_meta_bytes=458632 combine_payload_bytes=913375232 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=913833864 ingress_bytes=1618018304 link_bytes=1618018304 token_bytes=1827209096 weight_movement_bytes=704643072 total_nvlink_bytes=2531852168 weight_to_token_ratio=0.385639 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.060227108e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=931020800 dispatch_meta_bytes=454696 combine_payload_bytes=931020800 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=931475496 ingress_bytes=1283342336 link_bytes=1283342336 token_bytes=1862496296 weight_movement_bytes=352321536 total_nvlink_bytes=2214817832 weight_to_token_ratio=0.189166 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.018939 gemm_active_groups=11 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.186181388e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129524 payload_rows_remote=113168 dup_rows=1548 dup_rows_remote=1492 dedup_ratio=0.011810 router_tokens_per_expert=[8021, 8112, 8179, 8149, 14, 19, 18, 857, 7, 5, 8167, 27, 12, 8178, 0, 0, 0, 9, 9, 0, 2, 14, 7538, 0, 0, 0, 8182, 9, 5, 3, 0, 0] payload_rows_per_dst=[16224, 16380, 16356, 16380, 16330, 15104, 16382, 16368] dup_rows_per_dst=[0, 1478, 56, 0, 4, 4, 0, 6] combine_rows_per_src=[16224, 16380, 16356, 16380, 16330, 15104, 16382, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=123526 payload_rows_remote=107432 dup_rows=7546 dup_rows_remote=306 dedup_ratio=0.057571 router_tokens_per_expert=[8168, 7735, 8192, 8192, 0, 0, 0, 463, 0, 0, 8192, 21, 0, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8183, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16094, 9342, 16190, 16382, 16384, 16366, 16384, 16384] dup_rows_per_dst=[7240, 292, 2, 0, 0, 12, 0, 0] combine_rows_per_src=[16094, 9342, 16190, 16382, 16384, 16366, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=933773312 dispatch_meta_bytes=458752 combine_payload_bytes=933773312 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=934232064 ingress_bytes=1286094848 link_bytes=1286094848 token_bytes=1868005376 weight_movement_bytes=352321536 total_nvlink_bytes=2220326912 weight_to_token_ratio=0.188608 gemm_rows_defined=66048 gemm_rows_charged=67584 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.022727 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.201594306e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=12 export_slots=12 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 4, 2, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=25517093608 prefetch_bytes=5989466112 reduce_bytes=3699376128 weight_movement_bytes=9688842240 egress_bytes=12762133224 ingress_bytes=22443802624 link_bytes=22443802624 total_nvlink_bytes=35205935848 weight_to_token_ratio=0.379700 prefetch_share_of_ingress=0.266865 dedup_ratio=0.133313 prefetch_slots=68 unused_prefetch_slots=188 slot_utilization=0.265625 reduce_slots=21 remote_demand=34 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020123 gemm_inner_pad_upper_bound=20193 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.854745387e-05 prefetch_bytes_per_gemm_flop=2.096156881e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=26986182344 prefetch_bytes=5284823040 reduce_bytes=5989466112 weight_movement_bytes=11274289152 egress_bytes=13496661704 ingress_bytes=24763809792 link_bytes=24763809792 total_nvlink_bytes=38260471496 weight_to_token_ratio=0.417780 prefetch_share_of_ingress=0.213409 dedup_ratio=0.091830 prefetch_slots=60 unused_prefetch_slots=196 slot_utilization=0.234375 reduce_slots=34 remote_demand=30 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018584 gemm_inner_pad_upper_bound=22098 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.666687370e-05 prefetch_bytes_per_gemm_flop=1.849550189e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129926 payload_rows_remote=113548 dup_rows=1146 dup_rows_remote=1136 dedup_ratio=0.008743 router_tokens_per_expert=[8178, 7625, 8190, 8192, 0, 0, 0, 573, 0, 0, 8192, 8, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[15250, 16382, 16384, 16384, 16380, 16378, 16384, 16384] dup_rows_per_dst=[0, 1120, 16, 0, 0, 10, 0, 0] combine_rows_per_src=[15250, 16382, 16384, 16384, 16380, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=927072256 dispatch_meta_bytes=458640 combine_payload_bytes=927072256 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=927530896 ingress_bytes=1807876096 link_bytes=1807876096 token_bytes=1854603152 weight_movement_bytes=880803840 total_nvlink_bytes=2735406992 weight_to_token_ratio=0.474928 gemm_rows_defined=66432 gemm_rows_charged=67584 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.017045 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.012335149e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=880082944 dispatch_meta_bytes=430952 combine_payload_bytes=880082944 prefetch_bytes=352321536 reduce_bytes=2113929216 egress_bytes=880513896 ingress_bytes=3346333696 link_bytes=3346333696 token_bytes=1760596840 weight_movement_bytes=2466250752 total_nvlink_bytes=4226847592 weight_to_token_ratio=1.400804 gemm_rows_defined=65920 gemm_rows_charged=67584 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.024621 gemm_active_groups=8 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.873807187e-04 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28252351816 prefetch_bytes=5460983808 reduce_bytes=5284823040 weight_movement_bytes=10745806848 egress_bytes=14129868104 ingress_bytes=24868290560 link_bytes=24868290560 total_nvlink_bytes=38998158664 weight_to_token_ratio=0.380351 prefetch_share_of_ingress=0.219596 dedup_ratio=0.067182 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=30 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018821 gemm_inner_pad_upper_bound=21844 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.703252913e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.250000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 E=32 B=4 K=8 R=8 NvS=33784 hidden_size=4096 assignments=65536 payload_rows=129786 payload_rows_remote=113402 dup_rows=1286 dup_rows_remote=1286 dedup_ratio=0.009811 router_tokens_per_expert=[8177, 7555, 8192, 8192, 0, 0, 0, 629, 0, 0, 8192, 20, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[15110, 16376, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 1236, 40, 0, 0, 10, 0, 0] combine_rows_per_src=[15110, 16376, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=930185216 dispatch_meta_bytes=458736 combine_payload_bytes=930185216 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=930643952 ingress_bytes=1282506752 link_bytes=1282506752 token_bytes=1860829168 weight_movement_bytes=352321536 total_nvlink_bytes=2213150704 weight_to_token_ratio=0.189336 gemm_rows_defined=66304 gemm_rows_charged=67584 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.018939 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.181502467e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=27686416608 prefetch_bytes=4404019200 reduce_bytes=8455716864 weight_movement_bytes=12859736064 egress_bytes=13846835424 ingress_bytes=26699317248 link_bytes=26699317248 total_nvlink_bytes=40546152672 weight_to_token_ratio=0.464478 prefetch_share_of_ingress=0.164949 dedup_ratio=0.073342 prefetch_slots=50 unused_prefetch_slots=206 slot_utilization=0.195312 reduce_slots=48 remote_demand=25 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019176 gemm_inner_pad_upper_bound=22098 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.344064484e-05 prefetch_bytes_per_gemm_flop=1.541291824e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=25650763248 prefetch_bytes=5460983808 reduce_bytes=7927234560 weight_movement_bytes=13388218368 egress_bytes=12828857840 ingress_bytes=26210123776 link_bytes=26210123776 total_nvlink_bytes=39038981616 weight_to_token_ratio=0.521942 prefetch_share_of_ingress=0.208354 dedup_ratio=0.129342 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=45 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.018703 gemm_inner_pad_upper_bound=21463 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.172859531e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=8 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.500000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=928989184 dispatch_meta_bytes=458752 combine_payload_bytes=928989184 prefetch_bytes=704643072 reduce_bytes=0 egress_bytes=929447936 ingress_bytes=1633632256 link_bytes=1633632256 token_bytes=1858437120 weight_movement_bytes=704643072 total_nvlink_bytes=2563080192 weight_to_token_ratio=0.379159 gemm_rows_defined=66176 gemm_rows_charged=67584 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.020833 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.147658722e-05 prefetch_bytes_per_gemm_flop=3.945707071e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28052368896 prefetch_bytes=4932501504 reduce_bytes=1937768448 weight_movement_bytes=6870269952 egress_bytes=14029843968 ingress_bytes=20892794880 link_bytes=20892794880 total_nvlink_bytes=34922638848 weight_to_token_ratio=0.244909 prefetch_share_of_ingress=0.236086 dedup_ratio=0.063645 prefetch_slots=56 unused_prefetch_slots=200 slot_utilization=0.218750 reduce_slots=11 remote_demand=28 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.020715 gemm_inner_pad_upper_bound=19939 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.311933140e-05 prefetch_bytes_per_gemm_flop=1.726246843e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28036873168 prefetch_bytes=5460983808 reduce_bytes=7046430720 weight_movement_bytes=12507414528 egress_bytes=14022114256 ingress_bytes=26522173440 link_bytes=26522173440 total_nvlink_bytes=40544287696 weight_to_token_ratio=0.446106 prefetch_share_of_ingress=0.205903 dedup_ratio=0.067042 prefetch_slots=62 unused_prefetch_slots=194 slot_utilization=0.242188 reduce_slots=40 remote_demand=31 gemm_rows_charged=1081344 gemm_tail_pad_fraction=0.019058 gemm_inner_pad_upper_bound=21336 gemm_flops_step=285735584268288 bytes_per_gemm_flop=9.282068773e-05 prefetch_bytes_per_gemm_flop=1.911201862e-05 anomalies=none - [2026-09-16 21:37:10] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 572.4 | throughput per GPU (TFLOP/s/GPU): 582.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.846018E+00 | loss scale: 1.0 | grad norm: 5.017 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-09-16 21:37:10 -[rank0]:[W916 21:37:11.031142617 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W916 21:37:12.922480974 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W916 21:37:12.979667608 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W916 21:37:12.083183496 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W916 21:37:12.083970445 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W916 21:37:13.527329575 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W916 21:37:13.608793492 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W916 21:37:13.613765233 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0916 21:37:16.901000 34463 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0916 21:37:16.902000 34463 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 34649 closing signal SIGTERM -W0916 21:37:16.902000 34463 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 34650 closing signal SIGTERM -W0916 21:37:16.903000 34463 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 34651 closing signal SIGTERM -W0916 21:37:16.903000 34463 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 34652 closing signal SIGTERM -W0916 21:37:16.904000 34463 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 34653 closing signal SIGTERM -W0916 21:37:16.904000 34463 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 34654 closing signal SIGTERM -W0916 21:37:16.904000 34463 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 34656 closing signal SIGTERM -Traceback (most recent call last): - File "/usr/local/bin/torchrun", line 8, in - sys.exit(main()) - ^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper - return f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main - run(args) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run - elastic_launch( - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent - result = agent.run() - ^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper - result = f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run - result = self._invoke_run(role) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run - time.sleep(monitor_interval) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 34463 got signal: 15 +[21:15:47] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[21:15:47] WARN: MoonEP source is on 'megatron-integration', need 'megatron-ddp-symmetric-params' +[21:15:47] ERROR: MoonEP source must be /workspace/MoonEP @ megatron-ddp-symmetric-params diff --git a/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log b/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log index 591c9b6ad7320271542b14d0c176578d81c9ff19..20b621d3aa02c98b634cd9c110654348486fafff 100644 --- a/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log +++ b/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1/train.log @@ -1,5096 +1,3 @@ -[21:38:31] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -============================================================== - G3 / moonep (slots/rank=8 policy=matched) - E=64 K=16 H'=1792 m=8 - L=16 seq=4096 gbs=16 (2 microbatches) - cap_mode=none cap=none cg=0 recompute=none - memory_dump=none snapshots=/workspace/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe/tierbprobe_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ a6a1a221 -============================================================== -W0916 21:38:40.143000 40186 torch/distributed/run.py:766] -W0916 21:38:40.143000 40186 torch/distributed/run.py:766] ***************************************** -W0916 21:38:40.143000 40186 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0916 21:38:40.143000 40186 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.auto - attention_dropout ............................... 0.0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_scope ................................ full - cuda_graph_warmup_steps ......................... 3 - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 8 - data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_type ................................. single - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - delay_wgrad_compute ............................. False - deprecated_use_mcore_models ..................... True - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_bf16_reduced_precision_matmul ........... False - disable_mamba_mem_eff_path ...................... False - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - embedding_init_method_std ....................... None - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_experimental ............................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_msc ...................................... True - enable_one_logger ............................... True - encoder_num_layers .............................. 16 - encoder_seq_length .............................. 4096 - end_weight_decay ................................ 0.1 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000000 - eval_iters ...................................... 0 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 8 - expert_tensor_parallel_size ..................... 1 - export_force_local_attention .................... False - export_kd_cfg ................................... None - export_kd_teacher_ckpt_format ................... None - export_kd_teacher_load .......................... None - export_kv_cache_quant ........................... False - export_legacy_megatron .......................... False - export_model_type ............................... GPTModel - export_moe_apply_probs_on_input ................. False - export_qk_l2_norm ............................... False - export_quant_cfg ................................ None - export_real_quant_cfg ........................... None - export_te_mcore_model ........................... False - external_cuda_graph ............................. False - ffn_hidden_size ................................. 14336 - finetune ........................................ False - finetune_data_split ............................. train - finetune_hf_dataset ............................. None - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. e4m3 - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... mxfp8 - fp8_wgrad ....................................... True - fsdp_double_buffer .............................. False - global_batch_size ............................... 16 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... True - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - heterogeneous_layers_config_encoded_json ........ None - heterogeneous_layers_config_path ................ None - hidden_dropout .................................. 0.0 - hidden_size ..................................... 4096 - hierarchical_context_parallel_sizes ............. None - high_priority_stream_groups ..................... [] - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_dynamic_batching ...................... False - inference_dynamic_batching_buffer_guaranteed_fraction 0.2 - inference_dynamic_batching_buffer_overflow_factor None - inference_dynamic_batching_buffer_size_gb ....... 40.0 - inference_dynamic_batching_chunk_size ........... 256 - inference_dynamic_batching_max_requests_override None - inference_dynamic_batching_max_tokens_override .. None - inference_dynamic_batching_num_cuda_graphs ...... 16 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.01 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 4294967296 - inprocess_active_world_size ..................... 8 - inprocess_barrier_timeout ....................... 120 - inprocess_completion_timeout .................... 120 - inprocess_empty_cuda_cache ...................... False - inprocess_granularity ........................... node - inprocess_hard_timeout .......................... 90 - inprocess_heartbeat_interval .................... 30 - inprocess_heartbeat_timeout ..................... 60 - inprocess_last_call_wait ........................ 1 - inprocess_max_iterations ........................ None - inprocess_monitor_process_interval .............. 1.0 - inprocess_monitor_thread_interval ............... 1.0 - inprocess_progress_watchdog_interval ............ 1.0 - inprocess_restart ............................... False - inprocess_soft_timeout .......................... 60 - inprocess_termination_grace_time ................ 1 - is_hybrid_model ................................. False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kitchen_config_file ............................. None - kitchen_recipe_number ........................... None - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - lazy_mpu_init ................................... None - load ............................................ None - load_main_params_from_ckpt ...................... None - load_model_opt_format ........................... False - local_rank ...................................... 0 - log_energy ...................................... False - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. True - log_timers_to_tensorboard ....................... False - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 8 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. None - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 5 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - mamba_head_dim .................................. 64 - mamba_num_groups ................................ 8 - mamba_num_heads ................................. None - mamba_state_dim ................................. 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... False - max_position_embeddings ......................... 32768 - max_tokens_to_oom ............................... 12000 - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - mid_level_dataset_surplus ....................... 0.005 - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-05 - mlp_chunks_for_prefill .......................... 1 - mmap_bin_files .................................. True - mock_data ....................................... False - moe_apply_probs_on_input ........................ False - moe_aux_loss_coeff .............................. 0.01 - moe_deepep_num_sms .............................. 20 - moe_echo_enable_random_offloading ............... False - moe_echo_expert_dispatch_overlap ................ False - moe_echo_expert_dispatcher_type ................. hybridep - moe_echo_recompute_expert_dispatch .............. False - moe_enable_deepep ............................... False - moe_enable_echo ................................. False - moe_enable_hybridep ............................. False - moe_enable_moonep ............................... True - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. 1792 - moe_grouped_gemm ................................ True - moe_hybridep_num_sms ............................ 16 - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_moonep_async_grad_reduce .................... True - moe_moonep_num_sms .............................. None - moe_moonep_traffic_logging ...................... True - moe_moonep_traffic_logging_interval ............. 1 - moe_moonep_traffic_warmup_iters ................. 2 - moe_num_echo_experts ............................ None - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_received_token_capacity ..................... None - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ fp32 - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 16 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... flex - moe_token_drop_policy ........................... probs - moe_upcycling_granularity ....................... 1 - moe_use_device_initiated_grouped_gemm ........... True - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mrope_section ................................... None - mscale .......................................... 1.0 - mscale_all_dim .................................. 0.0 - mtp_loss_scaling_factor ......................... 0.1 - mtp_num_layers .................................. None - multi_latent_attention .......................... False - nccl_all_reduce_for_prefill ..................... False - nccl_communicator_config_path ................... None - nccl_ub ......................................... False - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_rope_freq .................................... None - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-05 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... 64 - num_layers ...................................... 16 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 2 - object_storage_cache_path ....................... None - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_grad_reduce ............................. True - overlap_moe_expert_parallel_comm ................ False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ True - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_layout .................. None - pipeline_model_parallel_size .................... 1 - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_l2_norm ...................................... False - qk_layernorm .................................... False - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... None - recompute_method ................................ None - recompute_modules ............................... None - recompute_num_layers ............................ None - record_memory_history ........................... False - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... validate_results - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - reuse_grad_buf_for_mxfp8_param_ag ............... False - rope_scaling_factor ............................. 8.0 - rope_type ....................................... None - rotary_base ..................................... 1000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - run_workload_inspector_server ................... False - sample_rate ..................................... 1.0 - save ............................................ None - save_interval ................................... None - save_retain_interval ............................ None - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - seq_length ...................................... 4096 - sequence_parallel ............................... False - sft ............................................. False - sft_tokenizer_prompt_format ..................... nemotron-h-aligned - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - squared_relu .................................... False - start_weight_decay .............................. 0.1 - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - suggested_communication_unit_size ............... None - swiglu .......................................... True - swin_backbone_type .............................. tiny - symmetric_ar_type ............................... None - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. None - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. GPT2BPETokenizer - torch_fsdp2_reshard_after_forward ............... True - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 12 - train_samples ................................... None - train_sync_interval ............................. None - transformer_impl ................................ transformer_engine - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... True - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_sharp ....................................... False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ False - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json - vocab_size ...................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.1 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 8 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> building GPT2BPETokenizer tokenizer ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") - > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS -> initializing torch distributed ... -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : -7 -Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 5 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : -7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 0[Gloo] Rank 0 is connected to 0 -0 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : -0 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - -[rank3]:[W916 21:38:56.175223318 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W916 21:38:56.179762649 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W916 21:38:56.179841179 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W916 21:38:56.179853135 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W916 21:38:56.179960405 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W916 21:38:56.179990446 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W916 21:38:56.180154832 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -> compiling dataset index builder ... -make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.037 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -[rank0]:[W916 21:38:56.339954328 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 4.028 seconds -time to initialize megatron (seconds): 7.393 -[after megatron is initialized] datetime: 2026-09-16 21:39:02 -building GPT model ... -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3906080768 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 -Params for bucket 1 (206045184 elements, 206045184 padded size): - module.output_layer.weight -Params for bucket 2 (42217472 elements, 42217472 padded size): - module.decoder.layers.15.mlp.router.weight - module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.self_attention.linear_qkv.weight - module.decoder.layers.15.self_attention.linear_proj.weight - module.decoder.final_layernorm.weight - module.decoder.layers.15.pre_mlp_layernorm.weight -Params for bucket 3 (42213376 elements, 42213376 padded size): - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.14.self_attention.linear_qkv.weight - module.decoder.layers.14.pre_mlp_layernorm.weight -Params for bucket 4 (42213376 elements, 42213376 padded size): - module.decoder.layers.13.pre_mlp_layernorm.weight - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.self_attention.linear_qkv.weight - module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.13.self_attention.linear_proj.weight -Params for bucket 5 (42213376 elements, 42213376 padded size): - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.12.self_attention.linear_qkv.weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.12.pre_mlp_layernorm.weight -Params for bucket 6 (42213376 elements, 42213376 padded size): - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.self_attention.linear_qkv.weight - module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.11.pre_mlp_layernorm.weight -Params for bucket 7 (42213376 elements, 42213376 padded size): - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.10.self_attention.linear_qkv.weight -Params for bucket 8 (42213376 elements, 42213376 padded size): - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_qkv.weight - module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight -Params for bucket 9 (42213376 elements, 42213376 padded size): - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.8.pre_mlp_layernorm.weight -Params for bucket 10 (42213376 elements, 42213376 padded size): - module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_proj.weight -Params for bucket 11 (42213376 elements, 42213376 padded size): - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.linear_proj.weight -Params for bucket 12 (42213376 elements, 42213376 padded size): - module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight -Params for bucket 13 (42213376 elements, 42213376 padded size): - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.4.pre_mlp_layernorm.weight -Params for bucket 14 (42213376 elements, 42213376 padded size): - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.pre_mlp_layernorm.weight -Params for bucket 15 (42213376 elements, 42213376 padded size): - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.2.pre_mlp_layernorm.weight -Params for bucket 16 (42213376 elements, 42213376 padded size): - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_qkv.weight -Params for bucket 17 (42213376 elements, 42213376 padded size): - module.decoder.layers.0.self_attention.linear_proj.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.weight -Params for bucket 18 (206045184 elements, 206045184 padded size): - module.embedding.word_embeddings.weight -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 64 -Params for bucket 1 (44040192 elements, 44040192 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight0 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 - module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight5 - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 - module.decoder.layers.15.mlp.experts.linear_fc2.weight4 -Params for bucket 2 (44040192 elements, 44040192 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc1.weight1 - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 -Params for bucket 3 (44040192 elements, 44040192 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight3 - module.decoder.layers.15.mlp.experts.linear_fc1.weight4 -Params for bucket 4 (44040192 elements, 44040192 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight5 - module.decoder.layers.15.mlp.experts.linear_fc1.weight6 - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 -Params for bucket 5 (44040192 elements, 44040192 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight4 - module.decoder.layers.14.mlp.experts.linear_fc2.weight1 - module.decoder.layers.14.mlp.experts.linear_fc2.weight0 - module.decoder.layers.14.mlp.experts.linear_fc2.weight3 - module.decoder.layers.14.mlp.experts.linear_fc2.weight5 -Params for bucket 6 (44040192 elements, 44040192 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight6 - module.decoder.layers.14.mlp.experts.linear_fc2.weight7 - module.decoder.layers.14.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 -Params for bucket 7 (44040192 elements, 44040192 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight2 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc1.weight4 -Params for bucket 8 (44040192 elements, 44040192 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight5 - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 - module.decoder.layers.14.mlp.experts.linear_fc1.weight7 -Params for bucket 9 (44040192 elements, 44040192 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight4 - module.decoder.layers.13.mlp.experts.linear_fc2.weight1 - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight2 - module.decoder.layers.13.mlp.experts.linear_fc2.weight3 - module.decoder.layers.13.mlp.experts.linear_fc2.weight5 -Params for bucket 10 (44040192 elements, 44040192 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight6 - module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.13.mlp.experts.linear_fc1.weight1 - module.decoder.layers.13.mlp.experts.linear_fc2.weight7 -Params for bucket 11 (44040192 elements, 44040192 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight2 - module.decoder.layers.13.mlp.experts.linear_fc1.weight4 - module.decoder.layers.13.mlp.experts.linear_fc1.weight3 -Params for bucket 12 (44040192 elements, 44040192 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight5 - module.decoder.layers.13.mlp.experts.linear_fc1.weight7 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 -Params for bucket 13 (44040192 elements, 44040192 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight3 - module.decoder.layers.12.mlp.experts.linear_fc2.weight5 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 -Params for bucket 14 (44040192 elements, 44040192 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight6 - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 - module.decoder.layers.12.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight7 -Params for bucket 15 (44040192 elements, 44040192 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 - module.decoder.layers.12.mlp.experts.linear_fc1.weight3 - module.decoder.layers.12.mlp.experts.linear_fc1.weight4 -Params for bucket 16 (44040192 elements, 44040192 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight5 - module.decoder.layers.12.mlp.experts.linear_fc1.weight6 - module.decoder.layers.12.mlp.experts.linear_fc1.weight7 -Params for bucket 17 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight2 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc2.weight3 - module.decoder.layers.11.mlp.experts.linear_fc2.weight5 -Params for bucket 18 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight7 -Params for bucket 19 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 - module.decoder.layers.11.mlp.experts.linear_fc1.weight4 - module.decoder.layers.11.mlp.experts.linear_fc1.weight3 -Params for bucket 20 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 - module.decoder.layers.11.mlp.experts.linear_fc1.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight6 -Params for bucket 21 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight4 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 -Params for bucket 22 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 -Params for bucket 23 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight3 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 -Params for bucket 24 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight7 -Params for bucket 25 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight2 - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc2.weight5 -Params for bucket 26 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 -Params for bucket 27 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 -Params for bucket 28 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight6 -Params for bucket 29 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 -Params for bucket 30 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight7 -Params for bucket 31 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc1.weight4 -Params for bucket 32 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 - module.decoder.layers.8.mlp.experts.linear_fc1.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 -Params for bucket 33 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 -Params for bucket 34 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 -Params for bucket 35 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 -Params for bucket 36 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight5 - module.decoder.layers.7.mlp.experts.linear_fc1.weight7 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 -Params for bucket 37 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 -Params for bucket 38 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight7 -Params for bucket 39 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 -Params for bucket 40 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 -Params for bucket 41 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 -Params for bucket 42 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 -Params for bucket 43 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 -Params for bucket 44 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight6 -Params for bucket 45 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 -Params for bucket 46 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 -Params for bucket 47 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 -Params for bucket 48 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 -Params for bucket 49 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 -Params for bucket 50 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 -Params for bucket 51 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 -Params for bucket 52 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 - module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 -Params for bucket 53 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 -Params for bucket 54 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 -Params for bucket 55 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 - module.decoder.layers.2.mlp.experts.linear_fc1.weight3 - module.decoder.layers.2.mlp.experts.linear_fc1.weight4 -Params for bucket 56 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 - module.decoder.layers.2.mlp.experts.linear_fc1.weight7 -Params for bucket 57 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 -Params for bucket 58 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 -Params for bucket 59 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc1.weight4 -Params for bucket 60 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 -Params for bucket 61 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 -Params for bucket 62 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 - module.decoder.layers.0.mlp.experts.linear_fc1.weight1 -Params for bucket 63 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight3 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 -Params for bucket 64 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight5 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 - module.decoder.layers.0.mlp.experts.linear_fc1.weight6 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-16 21:39:05 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 192 - validation: 0 - test: 0 -INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] -> building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) -INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers -INFO:megatron.core.datasets.indexed_dataset: Extract the document indices -INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 -INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 -INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices -INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy -INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy -INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy -INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 -> finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-09-16 21:39:05 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3416.94, 3425.47) - train/valid/test-data-iterators-setup ..........: (67.99, 70.60) -training ... -Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-09-16 21:39:05 -INFO:moonep.api:MoonEP comm buffer sizes: - config : R=8, S=4096, H=4096, E=64, K=16, NvS=67568, NvS_padded=67584 - sm_count : num_sms=32, num_sms_dedup=148 - hidden_buffer_size : 528.00 MiB - meta_buffer_size : 2.00 MiB - local_temp_buffer_size : 2.26 MiB - total_size : 532.26 MiB -[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [1024, 23296, 6528, 3072, 11648, 4480, 7936, 7936, 384, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[22, -1, -1, -1, -1, -1, -1, -1], [32, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [40, 45, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1]] -[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [21888, 640, 1280, 24448, 3840, 3072, 0, 128, 10752, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[36, -1, -1, -1, -1, -1, -1, -1], [6, 0, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [17, -1, -1, -1, -1, -1, -1, -1], [17, -1, -1, -1, -1, -1, -1, -1], [28, -1, -1, -1, -1, -1, -1, -1], [17, -1, -1, -1, -1, -1, -1, -1], [35, -1, -1, -1, -1, -1, -1, -1]] -[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [23168, 4480, 18048, 768, 4480, 128, 14592, 0, 256, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[57, -1, -1, -1, -1, -1, -1, -1], [43, 40, -1, -1, -1, -1, -1, -1], [46, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1], [7, 2, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1]] -[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [3200, 128, 29312, 8832, 1536, 512, 896, 17024, 4736, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[62, -1, -1, -1, -1, -1, -1, -1], [56, -1, -1, -1, -1, -1, -1, -1], [40, -1, -1, -1, -1, -1, -1, -1], [43, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1], [62, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1]] -WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-09-16 21:39:28] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 22657.2 | throughput per GPU (TFLOP/s/GPU): 14.7 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102347E+01 | loss scale: 1.0 | grad norm: 44.191 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 23.22 -Number of parameters in embedding layers in billions: 0.41 -Total number of parameters in billions: 23.63 -Number of parameters in most loaded shard in billions: 23.6320 -Theoretical memory footprints: weight and optimizer=169029.43 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 53034.5634765625 | max allocated: 53034.564453125 | reserved: 55690.0 | max reserved: 55690.0 - [2026-09-16 21:39:29] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 721.1 | throughput per GPU (TFLOP/s/GPU): 462.4 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101568E+01 | loss scale: 1.0 | grad norm: 45.944 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121044 payload_rows_remote=105828 dup_rows=141100 dup_rows_remote=122430 dedup_ratio=0.538254 router_tokens_per_expert=[145, 5208, 2696, 345, 3655, 78, 1680, 845, 3192, 251, 2243, 821, 1404, 1304, 132, 871, 310, 1494, 3176, 2372, 3847, 1355, 4675, 3751, 198, 3805, 1068, 2143, 2705, 690, 3863, 3931, 6309, 544, 1573, 1463, 374, 2476, 3965, 1408, 5142, 3309, 3158, 1224, 856, 3734, 2087, 1455, 570, 886, 1005, 2975, 921, 958, 2430, 196, 320, 1979, 4652, 3629, 2313, 836, 2875, 1197] payload_rows_per_dst=[14312, 15328, 15010, 15414, 16018, 14262, 15216, 15484] dup_rows_per_dst=[14992, 15614, 16622, 20202, 20206, 13486, 18670, 21308] combine_rows_per_src=[14312, 15328, 15010, 15414, 16018, 14262, 15216, 15484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120920 payload_rows_remote=105844 dup_rows=141224 dup_rows_remote=124536 dedup_ratio=0.538727 router_tokens_per_expert=[259, 5518, 2792, 315, 3494, 111, 1504, 833, 2654, 209, 2050, 646, 2178, 1323, 104, 1022, 516, 2325, 2686, 2268, 3947, 819, 5028, 3120, 157, 3069, 1446, 2731, 2383, 576, 2597, 4036, 6827, 806, 557, 688, 237, 2018, 4402, 2354, 5632, 3145, 2725, 1043, 1414, 4343, 2350, 1404, 556, 1318, 652, 2550, 797, 1125, 3836, 301, 425, 1449, 4759, 4194, 2230, 798, 2132, 1289] payload_rows_per_dst=[15590, 15126, 15076, 15284, 15132, 15094, 15270, 14348] dup_rows_per_dst=[24046, 15276, 16688, 18706, 15972, 17754, 18290, 14492] combine_rows_per_src=[15590, 15126, 15076, 15284, 15132, 15094, 15270, 14348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120252 payload_rows_remote=104506 dup_rows=141892 dup_rows_remote=117888 dedup_ratio=0.541275 router_tokens_per_expert=[217, 4465, 1674, 321, 4418, 118, 1425, 1066, 2631, 183, 2145, 1463, 1423, 1244, 201, 1051, 638, 2291, 3446, 2918, 3647, 880, 5952, 3208, 161, 3118, 891, 1797, 1736, 667, 2969, 3697, 6880, 814, 981, 446, 1064, 2425, 4263, 1806, 4545, 2551, 3632, 1345, 1616, 3810, 2140, 2297, 686, 973, 801, 2270, 574, 522, 4319, 578, 517, 2108, 3940, 4891, 1867, 1388, 1866, 1096] payload_rows_per_dst=[14328, 15396, 14812, 13314, 15318, 15602, 15736, 15746] dup_rows_per_dst=[13080, 16522, 15420, 10106, 17616, 22582, 22562, 24004] combine_rows_per_src=[14328, 15396, 14812, 13314, 15318, 15602, 15736, 15746] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120392 payload_rows_remote=104912 dup_rows=141752 dup_rows_remote=122128 dedup_ratio=0.540741 router_tokens_per_expert=[215, 5155, 2073, 445, 3459, 131, 2031, 930, 2726, 306, 2558, 804, 2060, 1370, 261, 1190, 349, 1577, 3424, 1910, 3982, 1205, 4089, 3500, 342, 3207, 1455, 1931, 2196, 740, 3565, 3745, 6056, 763, 1121, 1000, 507, 2502, 3954, 1174, 5269, 2959, 3225, 1274, 1024, 4024, 2228, 1327, 572, 815, 1305, 2915, 1306, 979, 3572, 282, 199, 2644, 4703, 3637, 2282, 1110, 1790, 1623] payload_rows_per_dst=[15774, 15480, 15530, 15204, 14756, 14450, 15114, 14084] dup_rows_per_dst=[24742, 19624, 20264, 19158, 12840, 15250, 17232, 12642] combine_rows_per_src=[15774, 15480, 15530, 15204, 14756, 14450, 15114, 14084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121478 payload_rows_remote=106344 dup_rows=140666 dup_rows_remote=122338 dedup_ratio=0.536598 router_tokens_per_expert=[115, 4896, 2979, 184, 3826, 120, 1955, 890, 1835, 177, 1584, 1233, 1801, 1191, 205, 1013, 311, 2130, 2528, 2284, 3578, 1101, 5263, 3481, 225, 3950, 1445, 2327, 1877, 612, 2359, 3793, 7126, 925, 667, 1564, 549, 2273, 3895, 2021, 5624, 2684, 2685, 1045, 1394, 4586, 2807, 1744, 649, 780, 730, 2382, 813, 742, 3693, 199, 635, 1499, 4005, 5123, 2166, 1025, 2151, 1623] payload_rows_per_dst=[14300, 15170, 15000, 15170, 16192, 15134, 14918, 15594] dup_rows_per_dst=[15630, 14022, 15826, 18006, 21848, 18328, 16146, 20860] combine_rows_per_src=[14300, 15170, 15000, 15170, 16192, 15134, 14918, 15594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122104 payload_rows_remote=107106 dup_rows=140040 dup_rows_remote=122698 dedup_ratio=0.534210 router_tokens_per_expert=[288, 4600, 1698, 238, 4595, 139, 1626, 841, 2225, 173, 2724, 1058, 1870, 1372, 230, 879, 331, 1768, 2531, 2300, 3637, 1228, 5299, 3664, 195, 3302, 930, 2093, 1720, 707, 3662, 3561, 6569, 794, 571, 1212, 483, 2558, 4159, 1375, 4931, 3265, 2918, 1223, 1454, 4207, 1932, 2587, 679, 613, 1392, 2295, 1320, 818, 3293, 388, 511, 2288, 5061, 4807, 2210, 773, 1873, 1029] payload_rows_per_dst=[14770, 15376, 15174, 14998, 16004, 15432, 14778, 15572] dup_rows_per_dst=[15592, 16388, 15744, 17342, 19438, 19740, 16576, 19220] combine_rows_per_src=[14770, 15376, 15174, 14998, 16004, 15432, 14778, 15572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=119254 payload_rows_remote=103618 dup_rows=142890 dup_rows_remote=120012 dedup_ratio=0.545082 router_tokens_per_expert=[240, 5069, 2351, 373, 2840, 161, 1666, 713, 2196, 346, 2167, 1599, 1840, 1351, 152, 1160, 577, 1734, 3466, 1945, 4700, 792, 5019, 3328, 155, 3869, 1123, 2511, 2020, 732, 2865, 3816, 6572, 1276, 791, 1295, 485, 2411, 4065, 1354, 5079, 2078, 2798, 1403, 1135, 4332, 2652, 1589, 775, 857, 828, 2497, 1038, 1351, 3051, 216, 893, 2105, 4640, 3614, 2085, 1446, 2054, 1431] payload_rows_per_dst=[15636, 15834, 15724, 15250, 14270, 14740, 14844, 12956] dup_rows_per_dst=[22878, 23436, 21508, 19488, 11912, 17068, 16514, 10086] combine_rows_per_src=[15636, 15834, 15724, 15250, 14270, 14740, 14844, 12956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=866942976 dispatch_meta_bytes=913032 combine_payload_bytes=866942976 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=867856008 ingress_bytes=1219264512 link_bytes=1219264512 token_bytes=1734798984 weight_movement_bytes=352321536 total_nvlink_bytes=2087120520 weight_to_token_ratio=0.203091 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.021780 gemm_active_groups=22 gemm_peak_group_rows=20608 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.827372321e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120854 payload_rows_remote=105038 dup_rows=141290 dup_rows_remote=122484 dedup_ratio=0.538979 router_tokens_per_expert=[127, 4009, 2342, 259, 3787, 112, 1863, 738, 2843, 214, 2699, 1537, 1552, 1833, 125, 1188, 349, 2643, 2646, 2612, 3814, 1029, 5015, 3780, 110, 3347, 645, 2649, 2024, 513, 3655, 4308, 7055, 861, 915, 1182, 569, 2289, 3017, 1423, 4800, 2877, 3105, 1192, 1338, 4501, 2091, 1507, 429, 991, 884, 2130, 854, 623, 3937, 261, 738, 1906, 4225, 4977, 1873, 909, 1936, 1310] payload_rows_per_dst=[13760, 15534, 15258, 15356, 15816, 15000, 14670, 15460] dup_rows_per_dst=[12714, 18018, 18488, 19146, 18806, 18222, 15608, 20288] combine_rows_per_src=[13760, 15534, 15258, 15356, 15816, 15000, 14670, 15460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=867074048 dispatch_meta_bytes=921520 combine_payload_bytes=867074048 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=867995568 ingress_bytes=1395556352 link_bytes=1395556352 token_bytes=1735069616 weight_movement_bytes=528482304 total_nvlink_bytes=2263551920 weight_to_token_ratio=0.304589 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.023674 gemm_active_groups=18 gemm_peak_group_rows=14208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.814533037e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=856113152 dispatch_meta_bytes=889576 combine_payload_bytes=856113152 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=857002728 ingress_bytes=1296515072 link_bytes=1296515072 token_bytes=1713115880 weight_movement_bytes=440401920 total_nvlink_bytes=2153517800 weight_to_token_ratio=0.257077 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=17152 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.259943211e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=871170048 dispatch_meta_bytes=914728 combine_payload_bytes=871170048 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=872084776 ingress_bytes=1399652352 link_bytes=1399652352 token_bytes=1743254824 weight_movement_bytes=528482304 total_nvlink_bytes=2271737128 weight_to_token_ratio=0.303158 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.022727 gemm_active_groups=18 gemm_peak_group_rows=13312 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.837468928e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=877412352 dispatch_meta_bytes=919216 combine_payload_bytes=877412352 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=878331568 ingress_bytes=1229733888 link_bytes=1229733888 token_bytes=1755743920 weight_movement_bytes=352321536 total_nvlink_bytes=2108065456 weight_to_token_ratio=0.200668 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=14080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.885996457e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=859439104 dispatch_meta_bytes=908160 combine_payload_bytes=859439104 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=860347264 ingress_bytes=1211760640 link_bytes=1211760640 token_bytes=1719786368 weight_movement_bytes=352321536 total_nvlink_bytes=2072107904 weight_to_token_ratio=0.204864 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.019886 gemm_active_groups=22 gemm_peak_group_rows=21120 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.785353770e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=848838656 dispatch_meta_bytes=894520 combine_payload_bytes=848838656 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=849733176 ingress_bytes=1024999424 link_bytes=1024999424 token_bytes=1698571832 weight_movement_bytes=176160768 total_nvlink_bytes=1874732600 weight_to_token_ratio=0.103711 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=19712 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.739568919e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=860471296 dispatch_meta_bytes=910088 combine_payload_bytes=860471296 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=861381384 ingress_bytes=1300873216 link_bytes=1300873216 token_bytes=1721852680 weight_movement_bytes=440401920 total_nvlink_bytes=2162254600 weight_to_token_ratio=0.255772 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=20992 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.284346998e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128798 payload_rows_remote=112418 dup_rows=133346 dup_rows_remote=106310 dedup_ratio=0.508675 router_tokens_per_expert=[8112, 13, 1, 472, 0, 463, 5, 68, 17, 8167, 346, 0, 0, 7, 137, 176, 0, 11, 3447, 8154, 7945, 8062, 0, 8182, 8170, 1, 1, 6, 2297, 10, 254, 3, 7076, 8163, 0, 8173, 16, 114, 0, 0, 1128, 31, 2847, 8098, 5, 2, 308, 1, 2, 3056, 41, 4456, 31, 5998, 1, 11, 12, 61, 1, 9, 7693, 7575, 37, 1599] payload_rows_per_dst=[16374, 16364, 14258, 16370, 16298, 16380, 16380, 16374] dup_rows_per_dst=[16062, 16724, 6524, 13018, 19732, 16650, 27036, 17600] combine_rows_per_src=[16374, 16364, 14258, 16370, 16298, 16380, 16380, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129580 payload_rows_remote=113202 dup_rows=132564 dup_rows_remote=112398 dedup_ratio=0.505692 router_tokens_per_expert=[8162, 0, 54, 873, 0, 41, 0, 6, 38, 8043, 224, 4, 13, 1, 93, 8, 1, 0, 5360, 8157, 8079, 7287, 0, 8114, 8033, 2, 7, 380, 2483, 1, 102, 58, 8165, 6447, 0, 8172, 107, 1832, 5, 0, 61, 56, 6065, 7015, 118, 1, 1033, 83, 1, 1398, 9, 5795, 16, 3009, 1, 6, 0, 27, 3, 76, 7919, 7995, 0, 33] payload_rows_per_dst=[16384, 16288, 16378, 16360, 16384, 16284, 15388, 16114] dup_rows_per_dst=[18102, 16766, 20166, 18956, 24920, 12580, 5082, 15992] combine_rows_per_src=[16384, 16288, 16378, 16360, 16384, 16284, 15388, 16114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125808 payload_rows_remote=109456 dup_rows=136336 dup_rows_remote=112454 dedup_ratio=0.520081 router_tokens_per_expert=[7558, 2, 63, 88, 17, 185, 73, 79, 70, 8177, 96, 69, 454, 108, 759, 54, 25, 32, 5576, 7986, 7953, 7759, 10, 8120, 7979, 0, 22, 1029, 243, 1, 17, 83, 4352, 7731, 62, 8124, 0, 708, 3, 1, 2016, 42, 6390, 7139, 278, 12, 155, 2, 16, 733, 7, 6111, 1, 6304, 0, 1, 24, 0, 33, 28, 8111, 7722, 0, 279] payload_rows_per_dst=[16384, 16368, 16332, 16362, 11498, 16352, 16130, 16382] dup_rows_per_dst=[15890, 19280, 11148, 11178, 14124, 23882, 24822, 16012] combine_rows_per_src=[16384, 16368, 16332, 16362, 11498, 16352, 16130, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129726 payload_rows_remote=113346 dup_rows=132418 dup_rows_remote=115718 dedup_ratio=0.505135 router_tokens_per_expert=[8173, 28, 0, 534, 1, 67, 49, 18, 15, 7698, 906, 0, 10, 330, 481, 304, 0, 1, 4127, 8160, 8170, 7769, 0, 8152, 8061, 1, 69, 533, 2363, 1, 1084, 1, 6044, 6229, 4, 8180, 0, 301, 40, 1, 36, 8, 4200, 7522, 5, 28, 100, 16, 0, 3866, 6, 5227, 1, 6175, 17, 0, 235, 1, 7, 13, 8131, 7500, 28, 45] payload_rows_per_dst=[16384, 15772, 16380, 16384, 15678, 16364, 16384, 16380] dup_rows_per_dst=[7002, 3716, 22870, 16000, 19894, 15658, 30578, 16700] combine_rows_per_src=[16384, 15772, 16380, 16384, 15678, 16364, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=920928256 dispatch_meta_bytes=874912 combine_payload_bytes=920928256 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=921803168 ingress_bytes=1097089024 link_bytes=1097089024 token_bytes=1842731424 weight_movement_bytes=176160768 total_nvlink_bytes=2018892192 weight_to_token_ratio=0.095598 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=22144 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.143240587e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129750 payload_rows_remote=113366 dup_rows=132394 dup_rows_remote=115796 dedup_ratio=0.505043 router_tokens_per_expert=[7630, 0, 0, 1019, 32, 25, 7, 23, 0, 8128, 325, 74, 0, 115, 18, 676, 16, 51, 6062, 8120, 7998, 7795, 0, 8077, 8021, 9, 2, 94, 175, 168, 9, 9, 4407, 8054, 20, 8153, 17, 188, 38, 139, 3119, 587, 1481, 4435, 2929, 2, 1388, 53, 0, 4492, 124, 6696, 3, 3364, 0, 0, 45, 0, 7, 174, 7955, 7902, 0, 622] payload_rows_per_dst=[16354, 16380, 16384, 16384, 16370, 15448, 16120, 16310] dup_rows_per_dst=[17468, 18438, 19542, 16598, 17470, 12540, 13238, 17100] combine_rows_per_src=[16354, 16380, 16384, 16384, 16370, 15448, 16120, 16310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 2, 0, 1, 2, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128316 payload_rows_remote=111932 dup_rows=133828 dup_rows_remote=116450 dedup_ratio=0.510513 router_tokens_per_expert=[7984, 0, 247, 91, 0, 64, 96, 268, 1, 7402, 145, 0, 4, 1, 2196, 33, 0, 7, 4218, 6950, 7935, 7327, 0, 8141, 8155, 20, 42, 1723, 2212, 337, 1168, 1, 3055, 7367, 10, 8029, 1, 1002, 291, 7, 923, 147, 5375, 7996, 195, 12, 426, 7, 223, 4278, 2, 5688, 0, 2900, 1, 278, 360, 0, 88, 97, 7737, 7700, 70, 39] payload_rows_per_dst=[16384, 16326, 16204, 16384, 15896, 16130, 15774, 15218] dup_rows_per_dst=[17378, 19220, 22122, 18800, 15760, 15348, 10966, 14234] combine_rows_per_src=[16384, 16326, 16204, 16384, 15896, 16130, 15774, 15218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129426 payload_rows_remote=113042 dup_rows=132718 dup_rows_remote=116018 dedup_ratio=0.506279 router_tokens_per_expert=[8127, 90, 3, 817, 39, 53, 104, 6, 32, 7907, 13, 1, 93, 2, 302, 52, 41, 2, 2421, 7608, 8163, 7939, 2, 8039, 8179, 1, 180, 428, 2621, 25, 145, 68, 6548, 6816, 3, 8187, 558, 10, 3, 9, 1122, 2, 8030, 7473, 14, 0, 784, 9, 12, 4454, 20, 4023, 1, 2396, 45, 4, 84, 22, 1, 690, 7677, 7850, 1, 721] payload_rows_per_dst=[16384, 16384, 16276, 16380, 16380, 16354, 14890, 16378] dup_rows_per_dst=[18104, 16700, 19864, 15100, 19702, 18514, 7020, 17714] combine_rows_per_src=[16384, 16384, 16276, 16380, 16380, 16354, 14890, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=927350784 dispatch_meta_bytes=902400 combine_payload_bytes=927350784 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=928253184 ingress_bytes=1808154624 link_bytes=1808154624 token_bytes=1855603968 weight_movement_bytes=880803840 total_nvlink_bytes=2736407808 weight_to_token_ratio=0.474672 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=28800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.012491113e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=896663552 dispatch_meta_bytes=887640 combine_payload_bytes=896663552 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=897551192 ingress_bytes=1072824320 link_bytes=1072824320 token_bytes=1794214744 weight_movement_bytes=176160768 total_nvlink_bytes=1970375512 weight_to_token_ratio=0.098183 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=29184 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.007368373e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=928530432 dispatch_meta_bytes=916256 combine_payload_bytes=928530432 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=929446688 ingress_bytes=1192771584 link_bytes=1192771584 token_bytes=1857977120 weight_movement_bytes=264241152 total_nvlink_bytes=2122218272 weight_to_token_ratio=0.142220 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.679022983e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127776 payload_rows_remote=113244 dup_rows=134368 dup_rows_remote=126862 dedup_ratio=0.512573 router_tokens_per_expert=[8180, 14, 1, 2099, 0, 211, 34, 67, 1, 7614, 657, 8, 206, 7, 2221, 52, 0, 0, 5466, 8140, 7932, 8142, 4, 8179, 7985, 17, 3, 212, 2134, 3, 329, 81, 4980, 6814, 0, 8190, 4, 51, 32, 4, 367, 7, 4764, 7771, 209, 0, 1179, 53, 14, 4839, 56, 2135, 51, 2722, 11, 2, 334, 1, 0, 171, 8165, 7073, 8, 1066] payload_rows_per_dst=[16384, 16382, 16338, 16382, 14532, 16234, 15150, 16374] dup_rows_per_dst=[21106, 21036, 11358, 21488, 7506, 17426, 17186, 17262] combine_rows_per_src=[16384, 16382, 16338, 16382, 14532, 16234, 15150, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=928694272 dispatch_meta_bytes=916648 combine_payload_bytes=928694272 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=929610920 ingress_bytes=1104855040 link_bytes=1104855040 token_bytes=1858305192 weight_movement_bytes=176160768 total_nvlink_bytes=2034465960 weight_to_token_ratio=0.094796 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.186727035e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=916946944 dispatch_meta_bytes=913528 combine_payload_bytes=916946944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=917860472 ingress_bytes=1093107712 link_bytes=1093107712 token_bytes=1834807416 weight_movement_bytes=176160768 total_nvlink_bytes=2010968184 weight_to_token_ratio=0.096010 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.120946902e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=926040064 dispatch_meta_bytes=916240 combine_payload_bytes=926040064 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=926956304 ingress_bytes=1102200832 link_bytes=1102200832 token_bytes=1852996368 weight_movement_bytes=176160768 total_nvlink_bytes=2029157136 weight_to_token_ratio=0.095068 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.024621 gemm_active_groups=20 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.171864578e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130434 payload_rows_remote=114050 dup_rows=131710 dup_rows_remote=115394 dedup_ratio=0.502434 router_tokens_per_expert=[0, 5751, 5175, 6, 6656, 0, 10, 178, 77, 0, 0, 4680, 8189, 7888, 8158, 51, 0, 2600, 0, 2, 4780, 3052, 0, 2, 328, 0, 5992, 0, 33, 8172, 5113, 22, 7489, 89, 147, 1585, 53, 8112, 2, 1, 8188, 2, 80, 1, 4, 49, 3, 1204, 2, 0, 8145, 0, 0, 14, 0, 0, 8182, 283, 0, 1, 8155, 0, 1, 2365] payload_rows_per_dst=[16384, 16376, 15892, 16384, 16256, 16382, 16384, 16376] dup_rows_per_dst=[19168, 13914, 8274, 14784, 18700, 18976, 16316, 21578] combine_rows_per_src=[16384, 16376, 15892, 16384, 16256, 16382, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=927694848 dispatch_meta_bytes=960424 combine_payload_bytes=927694848 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=928655272 ingress_bytes=1368096768 link_bytes=1368096768 token_bytes=1856350120 weight_movement_bytes=440401920 total_nvlink_bytes=2296752040 weight_to_token_ratio=0.237241 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=26368 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.660770829e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129086 payload_rows_remote=112702 dup_rows=133058 dup_rows_remote=115596 dedup_ratio=0.507576 router_tokens_per_expert=[0, 6223, 5959, 0, 7918, 0, 0, 1045, 507, 0, 0, 6373, 8181, 5222, 6127, 33, 0, 150, 0, 8, 7878, 584, 1, 142, 162, 16, 7014, 0, 1, 8180, 4, 11, 7920, 3656, 632, 38, 6, 8181, 1, 1, 8186, 110, 11, 4, 14, 0, 511, 165, 77, 293, 8083, 17, 2, 0, 28, 0, 8077, 3068, 0, 4, 8160, 0, 69, 2019] payload_rows_per_dst=[16362, 14606, 16384, 16380, 16206, 16384, 16384, 16380] dup_rows_per_dst=[25928, 7514, 17462, 14396, 23560, 17076, 16978, 10144] combine_rows_per_src=[16362, 14606, 16384, 16380, 16206, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126072 payload_rows_remote=109892 dup_rows=136072 dup_rows_remote=116998 dedup_ratio=0.519073 router_tokens_per_expert=[1, 253, 5100, 0, 6401, 130, 0, 70, 1028, 2, 1, 4895, 8143, 7246, 7941, 225, 0, 14, 0, 0, 5856, 3107, 19, 0, 76, 0, 4538, 0, 1, 8125, 4345, 40, 8077, 4898, 1881, 1471, 65, 7637, 789, 77, 7803, 20, 0, 0, 18, 71, 1642, 431, 0, 0, 7862, 17, 82, 101, 13, 1, 7757, 4645, 28, 0, 7955, 4, 95, 75] payload_rows_per_dst=[15468, 16376, 12520, 16376, 16384, 16180, 16384, 16384] dup_rows_per_dst=[8442, 16798, 7284, 13514, 33406, 19074, 16054, 21500] combine_rows_per_src=[15468, 16376, 12520, 16376, 16384, 16180, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=934297600 dispatch_meta_bytes=917776 combine_payload_bytes=934297600 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=935215376 ingress_bytes=1110458368 link_bytes=1110458368 token_bytes=1869512976 weight_movement_bytes=176160768 total_nvlink_bytes=2045673744 weight_to_token_ratio=0.094228 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.218103333e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128446 payload_rows_remote=112062 dup_rows=133698 dup_rows_remote=100266 dedup_ratio=0.510017 router_tokens_per_expert=[5, 987, 3437, 0, 7447, 4, 25, 66, 2, 0, 3, 7339, 8170, 8106, 7868, 0, 37, 4111, 6, 2, 7369, 841, 8, 0, 328, 452, 4168, 70, 0, 8171, 4404, 0, 8184, 361, 552, 2553, 7, 7815, 1, 0, 8144, 42, 7, 0, 2, 588, 524, 1570, 514, 4, 8116, 64, 3, 25, 7, 3, 8050, 1977, 9, 0, 8186, 5, 5, 328] payload_rows_per_dst=[16144, 16320, 16340, 14124, 16368, 16382, 16384, 16384] dup_rows_per_dst=[7798, 14582, 16478, 11174, 19214, 13592, 17428, 33432] combine_rows_per_src=[16144, 16320, 16340, 14124, 16368, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130092 payload_rows_remote=114052 dup_rows=132052 dup_rows_remote=111314 dedup_ratio=0.503738 router_tokens_per_expert=[20, 4417, 4419, 0, 4176, 0, 5, 57, 343, 13, 6, 7076, 8113, 6826, 3696, 277, 239, 6975, 0, 25, 7923, 317, 13, 266, 384, 161, 6568, 2, 1, 8181, 3748, 2, 8176, 1314, 177, 202, 43, 7864, 10, 9, 8124, 5, 101, 58, 7, 0, 0, 15, 190, 1081, 8177, 0, 4, 236, 0, 17, 7871, 521, 7, 33, 8161, 12, 1, 4407] payload_rows_per_dst=[16372, 16040, 16354, 16384, 15844, 16382, 16382, 16334] dup_rows_per_dst=[9816, 20738, 31232, 24074, 9190, 7826, 19254, 9922] combine_rows_per_src=[16372, 16040, 16354, 16384, 15844, 16382, 16382, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130790 payload_rows_remote=114476 dup_rows=131354 dup_rows_remote=120318 dedup_ratio=0.501076 router_tokens_per_expert=[0, 4057, 6250, 4, 6382, 7, 0, 226, 2146, 12, 28, 7540, 8187, 5728, 8089, 47, 19, 4897, 0, 6, 8190, 821, 0, 3, 4, 80, 5095, 0, 10, 8081, 405, 0, 4223, 2510, 80, 27, 4, 8130, 99, 0, 6576, 2435, 0, 37, 0, 393, 652, 168, 9, 3, 7381, 0, 0, 8, 45, 0, 7791, 1386, 0, 0, 7987, 0, 307, 4507] payload_rows_per_dst=[16378, 16192, 16382, 16314, 16382, 16376, 16384, 16382] dup_rows_per_dst=[17474, 15566, 26708, 11036, 13764, 19864, 14882, 12060] combine_rows_per_src=[16378, 16192, 16382, 16314, 16382, 16376, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125090 payload_rows_remote=109062 dup_rows=137054 dup_rows_remote=110908 dedup_ratio=0.522820 router_tokens_per_expert=[9, 3833, 7359, 0, 6339, 217, 13, 102, 973, 0, 0, 4567, 8190, 4394, 6435, 0, 43, 1280, 2, 172, 5628, 1635, 123, 27, 0, 175, 6987, 0, 10, 7757, 994, 122, 4793, 1692, 0, 1075, 53, 8106, 6, 2, 8190, 4, 0, 194, 2, 762, 750, 2981, 2988, 9, 8124, 1, 3, 6, 45, 16, 8021, 4050, 0, 1, 7221, 74, 560, 3957] payload_rows_per_dst=[16028, 16384, 16318, 16174, 11114, 16384, 16384, 16304] dup_rows_per_dst=[26146, 31456, 15054, 16840, 1696, 17236, 13196, 15430] combine_rows_per_src=[16028, 16384, 16318, 16174, 11114, 16384, 16384, 16304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=923254784 dispatch_meta_bytes=913192 combine_payload_bytes=923254784 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=924167976 ingress_bytes=1099415552 link_bytes=1099415552 token_bytes=1847422760 weight_movement_bytes=176160768 total_nvlink_bytes=2023583528 weight_to_token_ratio=0.095355 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.156268173e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=900235264 dispatch_meta_bytes=907560 combine_payload_bytes=900235264 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=901142824 ingress_bytes=1076396032 link_bytes=1076396032 token_bytes=1801378088 weight_movement_bytes=176160768 total_nvlink_bytes=1977538856 weight_to_token_ratio=0.097792 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.027368469e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=918011904 dispatch_meta_bytes=849312 combine_payload_bytes=918011904 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=918861216 ingress_bytes=1270333440 link_bytes=1270333440 token_bytes=1836873120 weight_movement_bytes=352321536 total_nvlink_bytes=2189194656 weight_to_token_ratio=0.191805 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.113337001e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=937787392 dispatch_meta_bytes=939176 combine_payload_bytes=937787392 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=938726568 ingress_bytes=1202028544 link_bytes=1202028544 token_bytes=1876513960 weight_movement_bytes=264241152 total_nvlink_bytes=2140755112 weight_to_token_ratio=0.140815 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.730858095e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=934313984 dispatch_meta_bytes=901464 combine_payload_bytes=934313984 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=935215448 ingress_bytes=1462796288 link_bytes=1462796288 token_bytes=1869529432 weight_movement_bytes=528482304 total_nvlink_bytes=2398011736 weight_to_token_ratio=0.282682 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.191048612e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=893435904 dispatch_meta_bytes=879880 combine_payload_bytes=893435904 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=894315784 ingress_bytes=1069596672 link_bytes=1069596672 token_bytes=1787751688 weight_movement_bytes=176160768 total_nvlink_bytes=1963912456 weight_to_token_ratio=0.098538 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=27776 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.989294892e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130708 payload_rows_remote=114428 dup_rows=131436 dup_rows_remote=113656 dedup_ratio=0.501389 router_tokens_per_expert=[0, 6822, 3751, 38, 6504, 0, 3, 0, 25, 0, 2, 6588, 8161, 6905, 8000, 93, 87, 1684, 6, 0, 7932, 8, 45, 0, 300, 2, 6818, 0, 1, 8056, 6918, 0, 8128, 20, 141, 645, 4, 8088, 1, 3, 8161, 97, 51, 249, 6, 0, 390, 50, 120, 211, 8111, 33, 1, 5, 8, 25, 8192, 176, 7, 186, 8092, 1, 1, 1120] payload_rows_per_dst=[16238, 16360, 16314, 16382, 16280, 16384, 16384, 16366] dup_rows_per_dst=[17998, 12922, 13148, 27808, 17780, 17646, 16966, 7168] combine_rows_per_src=[16238, 16360, 16314, 16382, 16280, 16384, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128722 payload_rows_remote=112578 dup_rows=133422 dup_rows_remote=128318 dedup_ratio=0.508965 router_tokens_per_expert=[1, 9, 5297, 8159, 78, 5447, 322, 0, 5815, 4, 0, 0, 9, 3243, 1, 0, 988, 7848, 47, 8162, 3, 4339, 48, 15, 8179, 9, 14, 0, 0, 3, 8058, 0, 8175, 18, 10, 3776, 7732, 1601, 16, 8142, 0, 0, 304, 1, 2, 0, 867, 0, 7955, 0, 0, 28, 31, 143, 848, 8142, 708, 10, 1, 0, 9, 7773, 527, 8155] payload_rows_per_dst=[14390, 16368, 16384, 16380, 16384, 16288, 16144, 16384] dup_rows_per_dst=[2066, 23952, 26510, 16146, 10808, 17808, 5104, 31028] combine_rows_per_src=[14390, 16368, 16384, 16380, 16384, 16288, 16144, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=937394176 dispatch_meta_bytes=912336 combine_payload_bytes=937394176 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=938306512 ingress_bytes=1553956864 link_bytes=1553956864 token_bytes=1875700688 weight_movement_bytes=616562688 total_nvlink_bytes=2492263376 weight_to_token_ratio=0.328711 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=27520 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.701509785e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 2, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130768 payload_rows_remote=114612 dup_rows=131376 dup_rows_remote=124118 dedup_ratio=0.501160 router_tokens_per_expert=[0, 0, 6958, 8174, 4, 7563, 225, 0, 700, 879, 0, 303, 0, 4161, 38, 8, 4, 7734, 0, 8180, 98, 3871, 0, 0, 8187, 0, 0, 0, 0, 0, 8189, 0, 7699, 291, 0, 5801, 8140, 0, 0, 8192, 0, 0, 967, 1, 1047, 0, 4296, 0, 8141, 0, 0, 18, 207, 62, 3066, 7984, 9, 26, 3, 0, 0, 1625, 59, 8162] payload_rows_per_dst=[16384, 16384, 16156, 16382, 16384, 16354, 16382, 16342] dup_rows_per_dst=[29478, 12140, 7258, 16370, 27582, 12548, 22574, 3426] combine_rows_per_src=[16384, 16384, 16156, 16382, 16384, 16354, 16382, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125032 payload_rows_remote=108676 dup_rows=137112 dup_rows_remote=117248 dedup_ratio=0.523041 router_tokens_per_expert=[1, 71, 4681, 7997, 0, 4538, 0, 39, 3285, 22, 0, 0, 1, 444, 156, 0, 2608, 7259, 195, 8135, 66, 6069, 5, 0, 8106, 0, 0, 0, 64, 532, 8032, 0, 6209, 1, 0, 3937, 7656, 118, 16, 7881, 53, 0, 26, 378, 2, 16, 4164, 0, 7476, 26, 0, 4, 399, 39, 7588, 8069, 2163, 0, 0, 57, 114, 4112, 98, 8164] payload_rows_per_dst=[10904, 16370, 16384, 16364, 16312, 16356, 15958, 16384] dup_rows_per_dst=[4430, 14978, 28078, 17104, 8382, 19864, 23212, 21064] combine_rows_per_src=[10904, 16370, 16384, 16364, 16312, 16356, 15958, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127276 payload_rows_remote=110892 dup_rows=134868 dup_rows_remote=109350 dedup_ratio=0.514481 router_tokens_per_expert=[13, 1, 8021, 8131, 41, 6183, 39, 0, 1508, 0, 1, 3, 0, 4287, 5, 1, 1550, 7963, 0, 8191, 31, 2129, 2, 187, 8109, 0, 6, 80, 0, 0, 7928, 38, 7935, 90, 0, 6608, 8124, 26, 2, 8159, 45, 0, 2381, 83, 33, 52, 40, 15, 7443, 0, 14, 0, 2, 165, 4456, 8181, 560, 0, 2, 8, 5, 3812, 330, 8053] payload_rows_per_dst=[16090, 14340, 16380, 16378, 16358, 16384, 14962, 16384] dup_rows_per_dst=[8230, 15008, 26526, 13436, 15472, 21480, 9198, 25518] combine_rows_per_src=[16090, 14340, 16380, 16378, 16358, 16384, 14962, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=922238976 dispatch_meta_bytes=963584 combine_payload_bytes=922238976 prefetch_bytes=0 reduce_bytes=616562688 egress_bytes=923202560 ingress_bytes=1538801664 link_bytes=1538801664 token_bytes=1845441536 weight_movement_bytes=616562688 total_nvlink_bytes=2462004224 weight_to_token_ratio=0.334100 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=27904 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.616646991e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128882 payload_rows_remote=112498 dup_rows=133262 dup_rows_remote=100228 dedup_ratio=0.508354 router_tokens_per_expert=[9, 0, 7477, 5801, 0, 8060, 135, 0, 120, 855, 360, 3391, 618, 4055, 47, 8, 6, 6872, 511, 7777, 38, 4841, 0, 254, 7946, 0, 588, 0, 3, 1, 8137, 96, 2500, 1843, 2, 4543, 7922, 920, 41, 7225, 1, 3, 876, 0, 1675, 11, 978, 0, 8160, 18, 3, 0, 117, 555, 4043, 8168, 2, 0, 1, 1, 30, 6237, 63, 7128] payload_rows_per_dst=[16384, 16384, 15926, 16384, 16380, 16134, 16344, 14946] dup_rows_per_dst=[33034, 18078, 17068, 18956, 18362, 6382, 9404, 11978] combine_rows_per_src=[16384, 16384, 15926, 16384, 16380, 16134, 16344, 14946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129888 payload_rows_remote=113504 dup_rows=132256 dup_rows_remote=116462 dedup_ratio=0.504517 router_tokens_per_expert=[0, 0, 5564, 7743, 29, 7811, 61, 0, 3818, 443, 0, 0, 9, 1412, 382, 0, 97, 7589, 0, 8032, 0, 3215, 0, 13, 8192, 0, 0, 0, 0, 0, 7896, 1, 6973, 12, 0, 3787, 7318, 0, 0, 8078, 0, 0, 303, 530, 4013, 0, 1339, 0, 8173, 0, 0, 0, 125, 168, 5417, 7942, 1683, 0, 1, 0, 1, 4395, 318, 8189] payload_rows_per_dst=[16238, 16384, 15348, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[22280, 15706, 6480, 15794, 14936, 17004, 27266, 12790] combine_rows_per_src=[16238, 16384, 15348, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128722 payload_rows_remote=112340 dup_rows=133422 dup_rows_remote=117484 dedup_ratio=0.508965 router_tokens_per_expert=[795, 1, 2224, 8055, 15, 4747, 2063, 7, 3773, 4, 24, 45, 44, 4095, 120, 0, 1208, 7095, 0, 8143, 4, 1456, 4, 445, 8143, 12, 88, 0, 29, 110, 8132, 48, 8179, 27, 0, 5810, 7420, 0, 2, 7750, 34, 0, 4064, 1, 39, 0, 3256, 0, 7696, 1, 2, 0, 0, 35, 3537, 8038, 1965, 24, 0, 34, 6, 4239, 86, 7898] payload_rows_per_dst=[16384, 16382, 14678, 16384, 16362, 16246, 16248, 16038] dup_rows_per_dst=[19606, 15938, 5746, 16740, 30588, 13232, 19106, 12466] combine_rows_per_src=[16384, 16382, 14678, 16384, 16362, 16246, 16248, 16038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938901504 dispatch_meta_bytes=954920 combine_payload_bytes=938901504 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939856424 ingress_bytes=1291223040 link_bytes=1291223040 token_bytes=1878757928 weight_movement_bytes=352321536 total_nvlink_bytes=2231079464 weight_to_token_ratio=0.187529 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230310041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=890273792 dispatch_meta_bytes=903696 combine_payload_bytes=890273792 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=891177488 ingress_bytes=1242595328 link_bytes=1242595328 token_bytes=1781451280 weight_movement_bytes=352321536 total_nvlink_bytes=2133772816 weight_to_token_ratio=0.197772 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.958015152e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=908427264 dispatch_meta_bytes=880968 combine_payload_bytes=908427264 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=909308232 ingress_bytes=1084588032 link_bytes=1084588032 token_bytes=1817735496 weight_movement_bytes=176160768 total_nvlink_bytes=1993896264 weight_to_token_ratio=0.096912 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.073240250e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=929824768 dispatch_meta_bytes=919864 combine_payload_bytes=929824768 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=930744632 ingress_bytes=1194065920 link_bytes=1194065920 token_bytes=1860569400 weight_movement_bytes=264241152 total_nvlink_bytes=2124810552 weight_to_token_ratio=0.142022 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.686270724e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=921583616 dispatch_meta_bytes=850904 combine_payload_bytes=921583616 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=922434520 ingress_bytes=1361985536 link_bytes=1361985536 token_bytes=1844018136 weight_movement_bytes=440401920 total_nvlink_bytes=2284420056 weight_to_token_ratio=0.238827 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=25856 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.626550481e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129684 payload_rows_remote=114664 dup_rows=132460 dup_rows_remote=126028 dedup_ratio=0.505295 router_tokens_per_expert=[494, 151, 6596, 7965, 34, 6969, 2, 3, 988, 2, 1, 1, 38, 3658, 69, 5, 3733, 8120, 4, 8113, 0, 4621, 12, 21, 8165, 5, 1, 97, 10, 7, 8173, 13, 4010, 23, 20, 3239, 8169, 0, 1, 7457, 11, 0, 4145, 7, 349, 0, 3134, 1, 7840, 9, 7, 12, 202, 35, 3100, 8166, 579, 31, 1, 0, 39, 4260, 0, 8154] payload_rows_per_dst=[16384, 16370, 16378, 16384, 15020, 16384, 16384, 16380] dup_rows_per_dst=[19862, 17362, 16844, 16558, 6432, 23296, 17474, 14632] combine_rows_per_src=[16384, 16370, 16378, 16384, 15020, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=920289280 dispatch_meta_bytes=919296 combine_payload_bytes=920289280 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=921208576 ingress_bytes=1184530432 link_bytes=1184530432 token_bytes=1841497856 weight_movement_bytes=264241152 total_nvlink_bytes=2105739008 weight_to_token_ratio=0.143493 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.019886 gemm_active_groups=21 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.632875972e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129474 payload_rows_remote=113090 dup_rows=132670 dup_rows_remote=114278 dedup_ratio=0.506096 router_tokens_per_expert=[1, 2, 2, 399, 9, 0, 22, 4116, 9, 0, 35, 7485, 5813, 1, 0, 8140, 304, 2862, 4, 9, 2800, 7970, 6718, 0, 0, 446, 37, 0, 0, 8106, 1, 2205, 8175, 8190, 142, 2998, 0, 0, 8169, 8146, 18, 1, 3840, 0, 8162, 0, 0, 0, 8, 974, 0, 13, 8190, 8175, 0, 28, 0, 20, 63, 19, 8186, 0, 48, 11] payload_rows_per_dst=[16368, 14990, 16228, 16384, 16356, 16380, 16384, 16384] dup_rows_per_dst=[9022, 11696, 9188, 19912, 15680, 32130, 18392, 16650] combine_rows_per_src=[16368, 14990, 16228, 16384, 16356, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=962768 combine_payload_bytes=939327488 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940290256 ingress_bytes=1467809792 link_bytes=1467809792 token_bytes=1879617744 weight_movement_bytes=528482304 total_nvlink_bytes=2408100048 weight_to_token_ratio=0.281165 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=27136 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219122141e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130606 payload_rows_remote=114222 dup_rows=131538 dup_rows_remote=108998 dedup_ratio=0.501778 router_tokens_per_expert=[356, 0, 0, 2, 0, 0, 1, 3804, 0, 0, 375, 7980, 7156, 8, 10, 8182, 0, 3187, 36, 122, 2610, 8032, 8172, 0, 0, 1, 25, 0, 2, 8173, 128, 234, 8177, 8076, 5976, 248, 0, 0, 8187, 8183, 424, 0, 3, 5, 7026, 0, 101, 0, 0, 1, 10, 2, 7952, 7593, 0, 10, 416, 832, 1, 1038, 8185, 4, 26, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16380, 15928, 16384] dup_rows_per_dst=[20690, 19770, 22540, 16906, 22878, 10226, 7922, 10606] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16380, 15928, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130478 payload_rows_remote=114094 dup_rows=131666 dup_rows_remote=108282 dedup_ratio=0.502266 router_tokens_per_expert=[0, 496, 6, 0, 9, 9, 1, 5177, 0, 281, 1, 7549, 7542, 0, 0, 7741, 377, 4072, 4, 272, 3954, 8054, 7940, 77, 198, 140, 0, 1, 2, 8186, 123, 178, 7516, 7401, 6921, 135, 270, 0, 8109, 7919, 2, 120, 695, 16, 8125, 0, 0, 9, 2, 112, 0, 0, 8168, 4752, 19, 22, 0, 4, 0, 16, 8167, 0, 182, 0] payload_rows_per_dst=[16384, 16208, 16380, 16384, 16012, 16384, 16382, 16344] dup_rows_per_dst=[16722, 14538, 25072, 17258, 8290, 23384, 9768, 16634] combine_rows_per_src=[16384, 16208, 16380, 16384, 16012, 16384, 16382, 16344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123678 payload_rows_remote=107294 dup_rows=138466 dup_rows_remote=119788 dedup_ratio=0.528206 router_tokens_per_expert=[2, 0, 0, 7, 0, 15, 3, 6982, 46, 709, 198, 7907, 6352, 0, 8, 8181, 0, 441, 14, 5, 83, 8022, 7375, 1, 32, 122, 3, 0, 0, 8048, 225, 14, 7431, 8125, 6657, 4, 1, 4, 8178, 8190, 1, 0, 293, 31, 8144, 1, 10, 1, 89, 3350, 1, 0, 8137, 8052, 0, 35, 92, 394, 484, 70, 8132, 0, 369, 1] payload_rows_per_dst=[16382, 14676, 11126, 16202, 16140, 16384, 16384, 16384] dup_rows_per_dst=[14010, 9368, 598, 8874, 28346, 25070, 33522, 18678] combine_rows_per_src=[16382, 14676, 11126, 16202, 16140, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=926433280 dispatch_meta_bytes=909472 combine_payload_bytes=926433280 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=927342752 ingress_bytes=1190674432 link_bytes=1190674432 token_bytes=1853776032 weight_movement_bytes=264241152 total_nvlink_bytes=2118017184 weight_to_token_ratio=0.142542 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.667279807e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129856 payload_rows_remote=113546 dup_rows=132288 dup_rows_remote=114976 dedup_ratio=0.504639 router_tokens_per_expert=[4, 0, 0, 1383, 0, 0, 2, 3987, 0, 0, 589, 8146, 7655, 0, 217, 8129, 892, 385, 0, 0, 2853, 7410, 5717, 0, 0, 129, 0, 0, 0, 7840, 13, 837, 7642, 8153, 3036, 1954, 0, 6, 8055, 8181, 6, 0, 591, 0, 7405, 0, 0, 0, 84, 3826, 0, 0, 7145, 6788, 0, 0, 16, 82, 3337, 335, 8181, 0, 61, 0] payload_rows_per_dst=[16384, 16382, 15260, 16310, 16382, 16384, 16370, 16384] dup_rows_per_dst=[17822, 16832, 11064, 17312, 9942, 16004, 19316, 23996] combine_rows_per_src=[16384, 16382, 15260, 16310, 16382, 16384, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129186 payload_rows_remote=112838 dup_rows=132958 dup_rows_remote=114894 dedup_ratio=0.507195 router_tokens_per_expert=[11, 1003, 21, 3, 15, 62, 4, 3836, 48, 148, 143, 7304, 3988, 0, 0, 6016, 867, 6994, 1, 0, 7983, 4467, 7993, 13, 30, 6, 2, 281, 14, 6252, 1653, 6, 4099, 5675, 8058, 107, 1, 538, 7821, 8165, 340, 0, 3473, 0, 4099, 0, 1, 33, 0, 296, 1, 259, 7931, 7440, 39, 0, 750, 278, 1217, 33, 8179, 0, 3075, 0] payload_rows_per_dst=[16348, 14838, 16384, 16384, 16348, 16262, 16238, 16384] dup_rows_per_dst=[18064, 23502, 37206, 7604, 20510, 7798, 7594, 10680] combine_rows_per_src=[16348, 14838, 16384, 16384, 16348, 16262, 16238, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=935706624 dispatch_meta_bytes=892880 combine_payload_bytes=935706624 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=936599504 ingress_bytes=1376108544 link_bytes=1376108544 token_bytes=1872306128 weight_movement_bytes=440401920 total_nvlink_bytes=2312708048 weight_to_token_ratio=0.235219 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=26624 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.705633430e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=934658048 dispatch_meta_bytes=889504 combine_payload_bytes=934658048 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=935547552 ingress_bytes=1198899200 link_bytes=1198899200 token_bytes=1870205600 weight_movement_bytes=264241152 total_nvlink_bytes=2134446752 weight_to_token_ratio=0.141290 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.713335075e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130574 payload_rows_remote=114192 dup_rows=131570 dup_rows_remote=105214 dedup_ratio=0.501900 router_tokens_per_expert=[0, 51, 17, 151, 17, 15, 1, 3042, 118, 170, 0, 7880, 4304, 39, 347, 8169, 31, 3459, 1, 9, 3932, 5266, 7509, 1, 0, 138, 70, 35, 0, 7325, 1246, 6813, 7765, 7656, 541, 51, 0, 47, 8187, 8105, 1180, 6, 3, 24, 5590, 0, 1, 4, 8, 3914, 24, 2, 8146, 7945, 0, 3, 263, 0, 209, 3043, 8186, 1, 12, 0] payload_rows_per_dst=[16384, 16382, 16384, 16382, 16030, 16244, 16384, 16384] dup_rows_per_dst=[22080, 26356, 23348, 23064, 8342, 5418, 15918, 7044] combine_rows_per_src=[16384, 16382, 16384, 16382, 16030, 16244, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=878952448 dispatch_meta_bytes=908328 combine_payload_bytes=878952448 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=879860776 ingress_bytes=1055113216 link_bytes=1055113216 token_bytes=1758813224 weight_movement_bytes=176160768 total_nvlink_bytes=1934973992 weight_to_token_ratio=0.100159 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.908193584e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=930168832 dispatch_meta_bytes=914088 combine_payload_bytes=930168832 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=931082920 ingress_bytes=1106329600 link_bytes=1106329600 token_bytes=1861251752 weight_movement_bytes=176160768 total_nvlink_bytes=2037412520 weight_to_token_ratio=0.094646 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.194983955e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=924368896 dispatch_meta_bytes=910928 combine_payload_bytes=924368896 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=925279824 ingress_bytes=1276690432 link_bytes=1276690432 token_bytes=1849648720 weight_movement_bytes=352321536 total_nvlink_bytes=2201970256 weight_to_token_ratio=0.190480 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.020833 gemm_active_groups=22 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.148933502e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130160 payload_rows_remote=113808 dup_rows=131984 dup_rows_remote=113826 dedup_ratio=0.503479 router_tokens_per_expert=[203, 127, 384, 1300, 0, 138, 0, 191, 5, 0, 47, 8188, 6202, 0, 669, 8188, 0, 1348, 0, 290, 33, 7756, 6734, 0, 11, 336, 0, 0, 1, 7982, 1680, 192, 8123, 7263, 7750, 2185, 28, 1, 8135, 8137, 2285, 1, 20, 6, 3323, 0, 67, 13, 0, 3982, 0, 0, 7302, 6537, 0, 0, 163, 84, 89, 2335, 8157, 3, 3077, 1] payload_rows_per_dst=[16384, 16384, 15806, 16384, 16352, 16348, 16118, 16384] dup_rows_per_dst=[12716, 13838, 9176, 20400, 18158, 11348, 19524, 26824] combine_rows_per_src=[16384, 16384, 15806, 16384, 16352, 16348, 16118, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=935460864 dispatch_meta_bytes=877624 combine_payload_bytes=935460864 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=936338488 ingress_bytes=1375862784 link_bytes=1375862784 token_bytes=1871799352 weight_movement_bytes=440401920 total_nvlink_bytes=2312201272 weight_to_token_ratio=0.235283 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.704257277e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[4, 0, 0, 1, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128566 payload_rows_remote=112182 dup_rows=133578 dup_rows_remote=103070 dedup_ratio=0.509560 router_tokens_per_expert=[0, 8179, 0, 0, 0, 0, 6, 27, 5597, 206, 0, 0, 2614, 8136, 0, 8167, 0, 240, 6, 8137, 0, 8029, 7833, 88, 0, 37, 0, 0, 8111, 2, 8190, 0, 4412, 0, 3854, 0, 0, 1543, 3658, 36, 0, 2, 6189, 10, 0, 7895, 1, 79, 4617, 0, 4165, 29, 17, 144, 20, 0, 8063, 0, 15, 29, 4831, 3, 0, 7855] payload_rows_per_dst=[16370, 15018, 16384, 16384, 16016, 16116, 16384, 15894] dup_rows_per_dst=[16114, 4834, 15146, 16296, 10990, 20316, 30508, 19374] combine_rows_per_src=[16370, 15018, 16384, 16384, 16016, 16116, 16384, 15894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=932315136 dispatch_meta_bytes=910536 combine_payload_bytes=932315136 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=933225672 ingress_bytes=1813118976 link_bytes=1813118976 token_bytes=1865540808 weight_movement_bytes=880803840 total_nvlink_bytes=2746344648 weight_to_token_ratio=0.472144 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=21632 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.015270943e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130652 payload_rows_remote=114358 dup_rows=131492 dup_rows_remote=115426 dedup_ratio=0.501602 router_tokens_per_expert=[245, 8175, 0, 0, 0, 3, 2, 1, 6655, 29, 32, 0, 2371, 8180, 43, 8120, 3, 2, 1220, 7251, 119, 8173, 5958, 289, 466, 19, 1, 1, 4917, 0, 8141, 0, 468, 47, 4064, 20, 0, 4005, 4083, 3536, 365, 0, 3528, 0, 58, 8122, 20, 0, 4822, 149, 2894, 653, 2, 30, 0, 13, 7979, 0, 0, 11, 7648, 3, 61, 8075] payload_rows_per_dst=[16384, 16362, 16298, 16384, 16242, 16294, 16384, 16304] dup_rows_per_dst=[16442, 10348, 14866, 10706, 16204, 16066, 23430, 23430] combine_rows_per_src=[16384, 16362, 16298, 16384, 16242, 16294, 16384, 16304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125892 payload_rows_remote=109508 dup_rows=136252 dup_rows_remote=113644 dedup_ratio=0.519760 router_tokens_per_expert=[61, 8169, 51, 10, 19, 0, 0, 269, 2296, 971, 0, 0, 4100, 8179, 0, 8188, 0, 2424, 773, 8192, 0, 8178, 8106, 1, 0, 0, 0, 0, 7756, 0, 8182, 0, 4835, 13, 6693, 0, 4, 1957, 25, 0, 0, 1, 6753, 50, 0, 8061, 2, 0, 4092, 2, 125, 256, 0, 16, 0, 1, 8172, 0, 0, 3, 6123, 0, 0, 7963] payload_rows_per_dst=[16384, 16384, 16384, 16384, 15342, 16384, 16384, 12246] dup_rows_per_dst=[17126, 22892, 22608, 22790, 12606, 21532, 8958, 7740] combine_rows_per_src=[16384, 16384, 16384, 16384, 15342, 16384, 16384, 12246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122352 payload_rows_remote=105968 dup_rows=139792 dup_rows_remote=101136 dedup_ratio=0.533264 router_tokens_per_expert=[50, 7826, 9, 0, 24, 16, 5, 12, 2699, 0, 9, 38, 3053, 8149, 0, 8168, 0, 1351, 4067, 8185, 0, 8175, 8043, 2, 121, 1, 1, 5, 7975, 62, 8182, 0, 4106, 0, 7152, 0, 13, 7702, 7, 5, 1, 15, 125, 1, 0, 8145, 0, 51, 1578, 0, 2097, 795, 0, 27, 136, 0, 8115, 0, 39, 0, 7284, 123, 3, 7324] payload_rows_per_dst=[16384, 8162, 16382, 16384, 15910, 16362, 16384, 16384] dup_rows_per_dst=[14386, 3436, 18334, 17076, 22060, 8422, 17422, 38656] combine_rows_per_src=[16384, 8162, 16382, 16384, 15910, 16362, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130982 payload_rows_remote=114598 dup_rows=131162 dup_rows_remote=115340 dedup_ratio=0.500343 router_tokens_per_expert=[0, 8114, 0, 0, 0, 0, 0, 104, 4158, 23, 0, 0, 4188, 8192, 1, 8142, 0, 42, 655, 7802, 0, 8088, 8180, 286, 2, 5, 0, 0, 7670, 234, 8192, 0, 4145, 7, 3775, 0, 1, 3412, 6717, 9, 4, 23, 2711, 1035, 0, 7835, 3, 0, 2509, 0, 71, 10, 3, 0, 0, 3, 8188, 0, 0, 2, 8192, 167, 0, 8172] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16370, 16312, 16384] dup_rows_per_dst=[16418, 23968, 17546, 15822, 23022, 15904, 5074, 13408] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16370, 16312, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=918994944 dispatch_meta_bytes=861008 combine_payload_bytes=918994944 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=919855952 ingress_bytes=1271316480 link_bytes=1271316480 token_bytes=1838850896 weight_movement_bytes=352321536 total_nvlink_bytes=2191172432 weight_to_token_ratio=0.191599 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.020833 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.118841614e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130900 payload_rows_remote=114524 dup_rows=131244 dup_rows_remote=114570 dedup_ratio=0.500656 router_tokens_per_expert=[0, 8110, 134, 0, 0, 0, 114, 106, 3853, 259, 2, 0, 2314, 8189, 35, 8104, 0, 3109, 0, 7980, 26, 8154, 6324, 2, 1164, 0, 124, 11, 5072, 0, 8173, 40, 5, 65, 2024, 23, 0, 4076, 4848, 1982, 0, 246, 2938, 1, 1, 6837, 1763, 4, 6122, 0, 3953, 163, 0, 154, 0, 0, 8033, 29, 24, 181, 7757, 4075, 39, 4330] payload_rows_per_dst=[16376, 16384, 16344, 16384, 16380, 16382, 16378, 16272] dup_rows_per_dst=[16674, 24660, 14840, 12784, 17832, 15334, 20708, 8412] combine_rows_per_src=[16376, 16384, 16344, 16384, 16380, 16382, 16378, 16272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=936820736 dispatch_meta_bytes=919136 combine_payload_bytes=936820736 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=937739872 ingress_bytes=1112981504 link_bytes=1112981504 token_bytes=1874560608 weight_movement_bytes=176160768 total_nvlink_bytes=2050721376 weight_to_token_ratio=0.093974 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.232231841e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130410 payload_rows_remote=114032 dup_rows=131734 dup_rows_remote=100234 dedup_ratio=0.502525 router_tokens_per_expert=[0, 8190, 1, 0, 0, 39, 1, 0, 5579, 239, 5, 58, 3854, 8187, 6, 8159, 8, 0, 465, 7930, 0, 7894, 7510, 29, 16, 0, 0, 14, 7486, 90, 8176, 0, 4585, 2, 5512, 0, 22, 4094, 4037, 76, 167, 0, 69, 630, 8, 7068, 5, 0, 3571, 0, 2803, 2, 39, 0, 1, 0, 8102, 14, 0, 7, 8087, 43, 0, 8192] payload_rows_per_dst=[16384, 16378, 16380, 16374, 16384, 15818, 16342, 16350] dup_rows_per_dst=[16428, 31500, 15504, 19492, 20272, 8112, 12282, 8144] combine_rows_per_src=[16384, 16378, 16380, 16374, 16384, 15818, 16342, 16350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=897089536 dispatch_meta_bytes=892608 combine_payload_bytes=897089536 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=897982144 ingress_bytes=1425571840 link_bytes=1425571840 token_bytes=1795071680 weight_movement_bytes=528482304 total_nvlink_bytes=2323553984 weight_to_token_ratio=0.294407 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.982607241e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=868089856 dispatch_meta_bytes=828416 combine_payload_bytes=868089856 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=868918272 ingress_bytes=1308491776 link_bytes=1308491776 token_bytes=1737008128 weight_movement_bytes=440401920 total_nvlink_bytes=2177410048 weight_to_token_ratio=0.253541 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.327007754e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938786816 dispatch_meta_bytes=919752 combine_payload_bytes=938786816 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939706568 ingress_bytes=1114947584 link_bytes=1114947584 token_bytes=1878493384 weight_movement_bytes=176160768 total_nvlink_bytes=2054654152 weight_to_token_ratio=0.093778 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243241068e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130468 payload_rows_remote=114432 dup_rows=131676 dup_rows_remote=121212 dedup_ratio=0.502304 router_tokens_per_expert=[3, 8041, 258, 0, 20, 0, 0, 592, 4769, 22, 0, 0, 4763, 8166, 3, 8160, 13, 200, 3, 8183, 118, 7975, 7232, 0, 0, 0, 79, 23, 8072, 2, 8072, 2, 7656, 0, 5286, 1, 0, 202, 47, 58, 0, 89, 6483, 6, 5, 7239, 1, 36, 891, 2, 4096, 36, 54, 39, 0, 0, 8180, 0, 113, 0, 7866, 8, 12, 7895] payload_rows_per_dst=[16384, 16358, 16384, 16372, 16036, 16384, 16384, 16166] dup_rows_per_dst=[17510, 19076, 12360, 16128, 10464, 26348, 12810, 16980] combine_rows_per_src=[16384, 16358, 16384, 16372, 16036, 16384, 16384, 16166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938180608 dispatch_meta_bytes=916376 combine_payload_bytes=938180608 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939096984 ingress_bytes=1114341376 link_bytes=1114341376 token_bytes=1877277592 weight_movement_bytes=176160768 total_nvlink_bytes=2053438360 weight_to_token_ratio=0.093838 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.023674 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239846557e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=934150144 dispatch_meta_bytes=857064 combine_payload_bytes=934150144 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=935007208 ingress_bytes=1638793216 link_bytes=1638793216 token_bytes=1869157352 weight_movement_bytes=704643072 total_nvlink_bytes=2573800424 weight_to_token_ratio=0.376984 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=25984 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.176557944e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130968 payload_rows_remote=114678 dup_rows=131176 dup_rows_remote=108464 dedup_ratio=0.500397 router_tokens_per_expert=[8167, 0, 830, 0, 8188, 0, 6550, 8177, 0, 0, 39, 371, 54, 0, 0, 33, 0, 108, 17, 7, 12, 0, 0, 0, 0, 3395, 8191, 5, 8174, 136, 16, 17, 1423, 1, 0, 0, 0, 1, 287, 8147, 0, 25, 8162, 5911, 38, 0, 0, 4359, 7, 8192, 8152, 0, 0, 0, 0, 8078, 8181, 14, 0, 8191, 3878, 5535, 3, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16384, 16290, 16378] dup_rows_per_dst=[14756, 11848, 15468, 15292, 11524, 12540, 22712, 27036] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16384, 16290, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=937426944 dispatch_meta_bytes=942576 combine_payload_bytes=937426944 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=938369520 ingress_bytes=1201668096 link_bytes=1201668096 token_bytes=1875796464 weight_movement_bytes=264241152 total_nvlink_bytes=2140037616 weight_to_token_ratio=0.140869 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=16256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.728839737e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129766 payload_rows_remote=114378 dup_rows=132378 dup_rows_remote=111500 dedup_ratio=0.504982 router_tokens_per_expert=[8007, 66, 0, 9, 7461, 30, 4403, 8109, 0, 38, 2, 0, 339, 206, 0, 28, 9, 14, 3, 0, 60, 113, 0, 1, 2, 591, 8098, 346, 4774, 117, 2, 1929, 0, 97, 7, 0, 0, 126, 1184, 7966, 3, 450, 7464, 7781, 0, 0, 0, 6425, 0, 8170, 8132, 0, 3617, 3, 95, 8153, 8097, 0, 12, 7923, 3810, 6795, 2, 3] payload_rows_per_dst=[16384, 16384, 16378, 16160, 16384, 15388, 16354, 16334] dup_rows_per_dst=[7554, 17078, 16622, 7548, 13822, 20878, 23372, 25504] combine_rows_per_src=[16384, 16384, 16378, 16160, 16384, 15388, 16354, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128958 payload_rows_remote=114080 dup_rows=133186 dup_rows_remote=107876 dedup_ratio=0.508064 router_tokens_per_expert=[8192, 0, 4068, 51, 8090, 4, 4774, 8184, 31, 66, 0, 0, 263, 63, 64, 1, 8, 550, 1, 7, 3, 0, 1, 16, 6, 312, 7995, 5222, 8080, 0, 23, 1349, 0, 62, 0, 1, 4, 1, 2733, 8189, 16, 6, 5087, 8120, 3, 44, 2, 4149, 0, 8151, 8186, 2, 1, 10, 93, 7806, 6809, 0, 0, 7803, 416, 5839, 25, 90] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 15786, 16378, 14878] dup_rows_per_dst=[17592, 9164, 9272, 14928, 10244, 14632, 32044, 25310] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 15786, 16378, 14878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130222 payload_rows_remote=113840 dup_rows=131922 dup_rows_remote=111004 dedup_ratio=0.503242 router_tokens_per_expert=[8165, 0, 10, 0, 8118, 0, 6318, 8175, 42, 0, 55, 0, 3987, 0, 0, 0, 0, 967, 0, 364, 6, 992, 0, 0, 0, 3589, 8169, 1035, 7486, 0, 2, 89, 108, 27, 17, 5, 22, 0, 2555, 5323, 5, 35, 8185, 8009, 10, 1, 0, 3674, 0, 8157, 8164, 0, 23, 1, 94, 8134, 7615, 0, 0, 8097, 778, 4418, 0, 46] payload_rows_per_dst=[16242, 16384, 16382, 15710, 16380, 16384, 16376, 16364] dup_rows_per_dst=[12650, 24464, 20918, 20136, 15954, 23454, 5022, 9324] combine_rows_per_src=[16242, 16384, 16382, 15710, 16380, 16384, 16376, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=892568 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940334744 ingress_bytes=1467924480 link_bytes=1467924480 token_bytes=1879776920 weight_movement_bytes=528482304 total_nvlink_bytes=2408259224 weight_to_token_ratio=0.281141 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219764346e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128290 payload_rows_remote=114562 dup_rows=133854 dup_rows_remote=128526 dedup_ratio=0.510612 router_tokens_per_expert=[8188, 0, 238, 6, 8192, 0, 7611, 8192, 253, 0, 33, 0, 2233, 4, 0, 0, 68, 41, 0, 99, 0, 12, 0, 0, 1, 6, 8169, 1, 5381, 0, 0, 47, 694, 877, 0, 0, 0, 0, 4059, 7569, 0, 174, 4097, 7872, 0, 1, 1, 7055, 0, 8192, 8185, 0, 1931, 0, 26, 8173, 7513, 0, 0, 7399, 4120, 3798, 27, 534] payload_rows_per_dst=[16384, 16384, 16384, 13728, 16382, 16282, 16366, 16380] dup_rows_per_dst=[15710, 21416, 16816, 5328, 23706, 22118, 12048, 16712] combine_rows_per_src=[16384, 16384, 16384, 13728, 16382, 16282, 16366, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130818 payload_rows_remote=114434 dup_rows=131326 dup_rows_remote=103638 dedup_ratio=0.500969 router_tokens_per_expert=[8181, 1126, 29, 0, 6707, 2, 1073, 8124, 2, 27, 331, 543, 227, 1, 0, 2, 14, 630, 309, 1205, 9, 315, 0, 22, 0, 4070, 4094, 1471, 7098, 13, 0, 3978, 301, 47, 18, 3, 0, 0, 211, 7960, 22, 40, 4812, 7889, 0, 12, 1, 5865, 252, 8191, 7835, 0, 4096, 1, 2316, 7391, 8175, 0, 35, 8063, 6, 7871, 40, 16] payload_rows_per_dst=[16384, 16384, 16384, 16348, 16372, 16338, 16306, 16302] dup_rows_per_dst=[27688, 13864, 13682, 26518, 16802, 13142, 11806, 7824] combine_rows_per_src=[16384, 16384, 16384, 16348, 16372, 16338, 16306, 16302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=936984576 dispatch_meta_bytes=903512 combine_payload_bytes=936984576 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=937888088 ingress_bytes=1377386496 link_bytes=1377386496 token_bytes=1874872664 weight_movement_bytes=440401920 total_nvlink_bytes=2315274584 weight_to_token_ratio=0.234897 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=23552 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.712789428e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=934543360 dispatch_meta_bytes=887824 combine_payload_bytes=934543360 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=935431184 ingress_bytes=1374945280 link_bytes=1374945280 token_bytes=1869974544 weight_movement_bytes=440401920 total_nvlink_bytes=2310376464 weight_to_token_ratio=0.235512 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.699119638e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=932577280 dispatch_meta_bytes=899376 combine_payload_bytes=932577280 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933476656 ingress_bytes=1284898816 link_bytes=1284898816 token_bytes=1866053936 weight_movement_bytes=352321536 total_nvlink_bytes=2218375472 weight_to_token_ratio=0.188806 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.020833 gemm_active_groups=22 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.194897026e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129072 payload_rows_remote=112688 dup_rows=133072 dup_rows_remote=116346 dedup_ratio=0.507629 router_tokens_per_expert=[7670, 19, 0, 0, 7735, 0, 5804, 7626, 359, 83, 0, 297, 0, 0, 17, 0, 21, 274, 0, 436, 0, 246, 191, 18, 0, 1, 8165, 415, 6052, 1, 0, 285, 2157, 0, 74, 49, 131, 0, 5481, 4438, 0, 1, 2201, 7531, 211, 708, 3, 6393, 497, 8189, 8150, 0, 4096, 1, 3, 7969, 8147, 2, 2, 6087, 7249, 5233, 313, 41] payload_rows_per_dst=[16352, 16384, 16384, 16372, 16378, 14782, 16252, 16168] dup_rows_per_dst=[18436, 16726, 17660, 15462, 24554, 17318, 8880, 14036] combine_rows_per_src=[16352, 16384, 16384, 16372, 16378, 14782, 16252, 16168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938491904 dispatch_meta_bytes=972352 combine_payload_bytes=938491904 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=939464256 ingress_bytes=1290813440 link_bytes=1290813440 token_bytes=1877956160 weight_movement_bytes=352321536 total_nvlink_bytes=2230277696 weight_to_token_ratio=0.187609 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228016452e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129990 payload_rows_remote=114140 dup_rows=132154 dup_rows_remote=117130 dedup_ratio=0.504128 router_tokens_per_expert=[8172, 2, 1136, 335, 8154, 0, 6998, 7959, 4, 0, 0, 11, 118, 9, 13, 16, 354, 1814, 0, 66, 134, 3, 4, 3, 0, 7102, 8172, 3496, 7636, 0, 0, 613, 243, 2, 37, 0, 17, 0, 926, 6322, 0, 20, 4396, 8135, 0, 0, 0, 4516, 0, 8115, 8167, 0, 15, 166, 17, 7967, 7891, 48, 0, 8184, 208, 2633, 721, 2] payload_rows_per_dst=[16382, 16384, 16384, 16260, 15850, 16384, 15962, 16384] dup_rows_per_dst=[16868, 16512, 20644, 29586, 15024, 10334, 15976, 7210] combine_rows_per_src=[16382, 16384, 16384, 16260, 15850, 16384, 15962, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=937443328 dispatch_meta_bytes=872288 combine_payload_bytes=937443328 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938315616 ingress_bytes=1465925632 link_bytes=1465925632 token_bytes=1875758944 weight_movement_bytes=528482304 total_nvlink_bytes=2404241248 weight_to_token_ratio=0.281743 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.208571632e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122614 payload_rows_remote=106230 dup_rows=139530 dup_rows_remote=131544 dedup_ratio=0.532265 router_tokens_per_expert=[38, 8191, 72, 2884, 14, 0, 8149, 0, 505, 0, 0, 0, 0, 0, 0, 0, 25, 7449, 7649, 5989, 3, 751, 0, 0, 1, 6085, 2, 33, 8, 3328, 8190, 3, 0, 8083, 8059, 99, 805, 8159, 2, 8191, 132, 8169, 0, 18, 126, 0, 1, 3559, 952, 64, 1, 0, 0, 4, 2, 0, 8169, 15, 119, 2, 0, 621, 8163, 8188] payload_rows_per_dst=[8502, 16382, 15816, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[2434, 17128, 17842, 26610, 17918, 24008, 7986, 25604] combine_rows_per_src=[8502, 16382, 15816, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=923140096 dispatch_meta_bytes=916136 combine_payload_bytes=923140096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=924056232 ingress_bytes=1275461632 link_bytes=1275461632 token_bytes=1847196328 weight_movement_bytes=352321536 total_nvlink_bytes=2199517864 weight_to_token_ratio=0.190733 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.017992 gemm_active_groups=22 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.142052735e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=935034880 dispatch_meta_bytes=925080 combine_payload_bytes=935034880 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=935959960 ingress_bytes=1111195648 link_bytes=1111195648 token_bytes=1870994840 weight_movement_bytes=176160768 total_nvlink_bytes=2047155608 weight_to_token_ratio=0.094154 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.222231793e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129500 payload_rows_remote=113400 dup_rows=132644 dup_rows_remote=108154 dedup_ratio=0.505997 router_tokens_per_expert=[13, 7911, 42, 3659, 3425, 0, 8110, 28, 80, 1, 0, 34, 0, 0, 96, 0, 0, 5079, 8108, 2531, 144, 4734, 117, 0, 0, 5499, 3, 1772, 0, 8, 5884, 0, 0, 7517, 8134, 8, 2983, 1548, 25, 8183, 37, 7935, 0, 15, 0, 35, 45, 5098, 3844, 299, 0, 0, 35, 10, 79, 32, 8192, 0, 1575, 1, 22, 2984, 8004, 7154] payload_rows_per_dst=[16244, 16370, 16380, 16364, 15274, 16100, 16384, 16384] dup_rows_per_dst=[7680, 16686, 15906, 14046, 8888, 24490, 16414, 28534] combine_rows_per_src=[16244, 16370, 16380, 16364, 15274, 16100, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124974 payload_rows_remote=108598 dup_rows=137170 dup_rows_remote=104110 dedup_ratio=0.523262 router_tokens_per_expert=[49, 7917, 327, 17, 5230, 10, 8170, 38, 924, 0, 13, 1, 0, 0, 0, 0, 6, 6911, 5444, 117, 13, 4962, 25, 0, 2, 8152, 0, 434, 11, 0, 6207, 9, 0, 8116, 8137, 0, 6716, 1597, 0, 8121, 501, 8058, 12, 0, 1, 0, 60, 5282, 598, 38, 133, 12, 20, 0, 0, 10, 8097, 0, 233, 0, 21, 4031, 8134, 8155] payload_rows_per_dst=[10668, 16384, 16112, 16368, 16298, 16384, 16384, 16376] dup_rows_per_dst=[674, 13676, 12152, 17842, 31050, 27278, 1438, 33060] combine_rows_per_src=[10668, 16384, 16112, 16368, 16298, 16384, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130552 payload_rows_remote=114174 dup_rows=131592 dup_rows_remote=112180 dedup_ratio=0.501984 router_tokens_per_expert=[0, 8142, 1129, 19, 4763, 0, 8087, 2, 6, 35, 2, 5, 1, 0, 42, 0, 1, 2585, 3479, 3997, 426, 8098, 0, 0, 70, 7926, 0, 2926, 0, 4, 8189, 1, 0, 7380, 8044, 103, 3896, 3190, 0, 8181, 15, 8190, 963, 0, 0, 17, 0, 2639, 2261, 67, 0, 11, 155, 0, 0, 13, 8176, 0, 36, 0, 47, 1415, 8192, 8146] payload_rows_per_dst=[16340, 16384, 16378, 16384, 15918, 16384, 16384, 16380] dup_rows_per_dst=[27944, 16248, 19412, 21848, 13220, 7264, 21366, 4290] combine_rows_per_src=[16340, 16384, 16378, 16384, 15918, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=870236160 dispatch_meta_bytes=951096 combine_payload_bytes=870236160 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=871187256 ingress_bytes=1222557696 link_bytes=1222557696 token_bytes=1741423416 weight_movement_bytes=352321536 total_nvlink_bytes=2093744952 weight_to_token_ratio=0.202318 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.021780 gemm_active_groups=22 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.845812777e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130098 payload_rows_remote=113806 dup_rows=132046 dup_rows_remote=126418 dedup_ratio=0.503716 router_tokens_per_expert=[0, 8190, 249, 4356, 1546, 0, 8188, 1, 0, 0, 0, 2, 0, 33, 3, 0, 0, 1873, 2832, 3882, 46, 7726, 0, 0, 0, 5473, 0, 970, 0, 0, 4517, 0, 0, 7871, 8192, 0, 8076, 4716, 0, 8192, 3, 8192, 484, 0, 0, 0, 0, 5418, 3201, 55, 0, 0, 338, 0, 0, 1, 8189, 0, 11, 0, 113, 4064, 8192, 5877] payload_rows_per_dst=[15920, 16384, 16094, 16292, 16352, 16384, 16384, 16288] dup_rows_per_dst=[22736, 16460, 16624, 5628, 24974, 15756, 23568, 6300] combine_rows_per_src=[15920, 16384, 16094, 16292, 16352, 16384, 16384, 16288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130726 payload_rows_remote=114344 dup_rows=131418 dup_rows_remote=88826 dedup_ratio=0.501320 router_tokens_per_expert=[31, 7869, 4095, 3755, 5905, 79, 7209, 0, 172, 6, 3, 493, 5, 0, 58, 0, 2, 4071, 4916, 4036, 8, 4074, 5, 26, 0, 8152, 449, 308, 23, 6, 6252, 963, 25, 7813, 8085, 25, 7094, 644, 12, 8082, 4, 7628, 1223, 0, 0, 17, 34, 1216, 0, 452, 2, 0, 50, 10, 13, 0, 8188, 2, 669, 2, 0, 1612, 7704, 7495] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16124, 16370, 16382, 16316] dup_rows_per_dst=[42592, 17424, 16804, 15922, 15102, 3874, 16456, 3244] combine_rows_per_src=[16382, 16384, 16384, 16384, 16124, 16370, 16382, 16316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=928972800 dispatch_meta_bytes=886216 combine_payload_bytes=928972800 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=929859016 ingress_bytes=1105133568 link_bytes=1105133568 token_bytes=1858831816 weight_movement_bytes=176160768 total_nvlink_bytes=2034992584 weight_to_token_ratio=0.094770 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.188286675e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=889634816 dispatch_meta_bytes=850832 combine_payload_bytes=889634816 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=890485648 ingress_bytes=1418117120 link_bytes=1418117120 token_bytes=1780120464 weight_movement_bytes=528482304 total_nvlink_bytes=2308602768 weight_to_token_ratio=0.296880 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.940863921e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=932298752 dispatch_meta_bytes=960896 combine_payload_bytes=932298752 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=933259648 ingress_bytes=1108459520 link_bytes=1108459520 token_bytes=1865558400 weight_movement_bytes=176160768 total_nvlink_bytes=2041719168 weight_to_token_ratio=0.094428 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.206910618e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=935313408 dispatch_meta_bytes=905416 combine_payload_bytes=935313408 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=936218824 ingress_bytes=1375715328 link_bytes=1375715328 token_bytes=1871532232 weight_movement_bytes=440401920 total_nvlink_bytes=2311934152 weight_to_token_ratio=0.235316 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=20352 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.703431585e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129678 payload_rows_remote=113294 dup_rows=132466 dup_rows_remote=115484 dedup_ratio=0.505318 router_tokens_per_expert=[1, 7843, 93, 3911, 3579, 0, 7725, 0, 305, 0, 3, 11, 0, 0, 22, 50, 0, 4400, 4681, 3745, 50, 25, 40, 263, 0, 7310, 803, 1, 16, 836, 4226, 32, 36, 6851, 8125, 428, 8146, 4096, 1, 8167, 154, 6954, 0, 0, 0, 1, 0, 4231, 4199, 50, 8, 61, 0, 70, 53, 642, 8117, 0, 0, 5, 0, 4796, 8144, 7766] payload_rows_per_dst=[15976, 16384, 16056, 16380, 16384, 15752, 16384, 16362] dup_rows_per_dst=[30328, 16982, 10352, 10068, 22732, 6928, 26304, 8772] combine_rows_per_src=[15976, 16384, 16056, 16380, 16384, 15752, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124908 payload_rows_remote=110428 dup_rows=137236 dup_rows_remote=136184 dedup_ratio=0.523514 router_tokens_per_expert=[19, 8155, 70, 3622, 2163, 12, 7956, 14, 112, 1, 17, 0, 15, 11, 0, 113, 0, 7689, 4135, 4010, 2, 8090, 0, 0, 3, 7347, 375, 1925, 0, 0, 8055, 107, 0, 7240, 8059, 0, 132, 394, 0, 8106, 3, 8188, 14, 5, 0, 0, 1, 7120, 2209, 0, 2, 31, 2, 3, 0, 9, 7954, 8, 0, 0, 65, 2069, 8179, 7261] payload_rows_per_dst=[12260, 16274, 16362, 16384, 14480, 16384, 16384, 16380] dup_rows_per_dst=[15464, 16594, 23470, 19240, 1052, 22404, 17542, 21470] combine_rows_per_src=[12260, 16274, 16362, 16384, 14480, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=936706048 dispatch_meta_bytes=812680 combine_payload_bytes=936706048 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=937518728 ingress_bytes=1553268736 link_bytes=1553268736 token_bytes=1874224776 weight_movement_bytes=616562688 total_nvlink_bytes=2490787464 weight_to_token_ratio=0.328969 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=26368 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.697656555e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124302 payload_rows_remote=107918 dup_rows=137842 dup_rows_remote=116996 dedup_ratio=0.525826 router_tokens_per_expert=[8, 4004, 8190, 8060, 1, 0, 0, 0, 0, 0, 2, 8182, 0, 0, 0, 11, 0, 0, 0, 8148, 77, 8125, 3812, 3, 8189, 0, 7, 0, 8158, 59, 8160, 0, 37, 8147, 24, 0, 7369, 30, 4905, 0, 5845, 0, 141, 1, 8187, 0, 8119, 0, 0, 119, 48, 2, 6, 3, 8192, 2877, 38, 8, 0, 8, 3744, 0, 4, 22] payload_rows_per_dst=[16220, 16384, 16352, 16384, 9812, 16384, 16384, 16382] dup_rows_per_dst=[7926, 16386, 20166, 16364, 12184, 20080, 20846, 23890] combine_rows_per_src=[16220, 16384, 16352, 16384, 9812, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=928104448 dispatch_meta_bytes=915112 combine_payload_bytes=928104448 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=929019560 ingress_bytes=1280425984 link_bytes=1280425984 token_bytes=1857124008 weight_movement_bytes=352321536 total_nvlink_bytes=2209445544 weight_to_token_ratio=0.189714 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.019886 gemm_active_groups=21 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.169851034e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=904626176 dispatch_meta_bytes=986448 combine_payload_bytes=904626176 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=905612624 ingress_bytes=1433108480 link_bytes=1433108480 token_bytes=1810238800 weight_movement_bytes=528482304 total_nvlink_bytes=2338721104 weight_to_token_ratio=0.291941 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.024809279e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129552 payload_rows_remote=113456 dup_rows=132592 dup_rows_remote=116912 dedup_ratio=0.505798 router_tokens_per_expert=[0, 6115, 8189, 8142, 1, 89, 8, 39, 215, 0, 0, 6462, 46, 0, 199, 0, 1, 0, 56, 7773, 22, 8079, 5100, 7, 7236, 1, 0, 0, 7981, 177, 6751, 39, 258, 8176, 6, 0, 5747, 27, 3810, 59, 8013, 9, 6, 1, 7584, 0, 7788, 3, 6, 0, 539, 0, 880, 901, 8125, 578, 69, 0, 1, 92, 5408, 0, 0, 258] payload_rows_per_dst=[16384, 16384, 16384, 15566, 15972, 16096, 16382, 16384] dup_rows_per_dst=[12404, 13836, 25692, 13966, 10882, 15680, 14648, 25484] combine_rows_per_src=[16384, 16384, 16384, 15566, 15972, 16096, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125904 payload_rows_remote=109520 dup_rows=136240 dup_rows_remote=117604 dedup_ratio=0.519714 router_tokens_per_expert=[15, 8063, 7742, 8150, 1, 0, 11, 7, 0, 1, 0, 8190, 3, 2, 0, 0, 0, 0, 0, 6234, 0, 8192, 2941, 1, 8123, 10, 0, 0, 8179, 41, 8173, 58, 0, 8159, 0, 0, 7895, 0, 244, 0, 8150, 0, 0, 101, 8187, 2, 8069, 3, 1, 0, 4, 123, 73, 694, 8188, 5875, 9, 5, 0, 0, 972, 126, 36, 19] payload_rows_per_dst=[16382, 16380, 14478, 16382, 13130, 16384, 16384, 16384] dup_rows_per_dst=[11172, 19546, 5514, 12206, 8106, 31170, 29890, 18636] combine_rows_per_src=[16382, 16380, 14478, 16382, 13130, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=884064256 dispatch_meta_bytes=899656 combine_payload_bytes=884064256 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=884963912 ingress_bytes=1060225024 link_bytes=1060225024 token_bytes=1769028168 weight_movement_bytes=176160768 total_nvlink_bytes=1945188936 weight_to_token_ratio=0.099581 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.936817575e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130958 payload_rows_remote=114574 dup_rows=131186 dup_rows_remote=118872 dedup_ratio=0.500435 router_tokens_per_expert=[3, 7839, 8192, 8187, 0, 19, 37, 1, 0, 0, 0, 8091, 122, 0, 0, 10, 0, 0, 138, 6127, 0, 8192, 2946, 44, 8191, 0, 0, 0, 8192, 9, 8192, 0, 2, 7781, 0, 1, 7178, 0, 5233, 0, 8183, 0, 0, 0, 6420, 0, 5767, 355, 3, 0, 0, 0, 0, 3, 8192, 3101, 0, 0, 287, 0, 4034, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16284, 16376, 16384, 16380] dup_rows_per_dst=[21196, 16446, 13104, 12314, 16388, 12792, 13932, 25014] combine_rows_per_src=[16384, 16384, 16382, 16384, 16284, 16376, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=118448 dedup_ratio=0.500092 router_tokens_per_expert=[0, 6446, 8192, 8134, 0, 338, 7, 11, 0, 0, 0, 8175, 93, 0, 1, 0, 0, 53, 0, 8187, 48, 8164, 2132, 0, 8177, 0, 0, 0, 8190, 0, 8152, 0, 22, 8120, 7, 26, 7742, 0, 3055, 0, 7625, 0, 56, 12, 8172, 57, 8141, 23, 0, 3, 0, 1238, 0, 40, 8185, 4023, 0, 0, 0, 0, 23, 2, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16364, 16384, 16384, 16384, 16380] dup_rows_per_dst=[29190, 16538, 12648, 20154, 14040, 13898, 18114, 6514] combine_rows_per_src=[16384, 16384, 16384, 16364, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130670 payload_rows_remote=114290 dup_rows=131474 dup_rows_remote=103848 dedup_ratio=0.501534 router_tokens_per_expert=[8, 3573, 7893, 6187, 1, 4, 0, 40, 0, 2, 5, 6376, 895, 81, 1, 1053, 0, 21, 1, 8164, 12, 8097, 4033, 33, 4552, 0, 34, 49, 8090, 0, 7855, 104, 1695, 7797, 1, 402, 6035, 15, 4277, 605, 3878, 5, 4056, 3, 5270, 416, 8170, 225, 0, 20, 2289, 3047, 0, 459, 8156, 3881, 91, 283, 1746, 241, 484, 0, 360, 1] payload_rows_per_dst=[16380, 15990, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[27626, 16626, 8148, 16978, 25270, 11404, 19320, 6102] combine_rows_per_src=[16380, 15990, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=929431552 dispatch_meta_bytes=921472 combine_payload_bytes=929431552 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=930353024 ingress_bytes=1369833472 link_bytes=1369833472 token_bytes=1859784576 weight_movement_bytes=440401920 total_nvlink_bytes=2300186496 weight_to_token_ratio=0.236803 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.670495647e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=897187840 dispatch_meta_bytes=908496 combine_payload_bytes=897187840 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=898096336 ingress_bytes=1249509376 link_bytes=1249509376 token_bytes=1795284176 weight_movement_bytes=352321536 total_nvlink_bytes=2147605712 weight_to_token_ratio=0.196248 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.019886 gemm_active_groups=22 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.996730935e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129500 payload_rows_remote=113116 dup_rows=132644 dup_rows_remote=116518 dedup_ratio=0.505997 router_tokens_per_expert=[0, 2856, 8161, 7759, 0, 0, 0, 75, 3, 26, 0, 8034, 27, 9, 6, 4, 1, 0, 86, 5128, 16, 8102, 6609, 0, 8067, 135, 2, 0, 7719, 118, 8176, 3, 0, 7597, 0, 226, 8176, 15, 8190, 21, 4827, 0, 18, 30, 7531, 37, 8024, 247, 1, 2, 175, 484, 169, 8, 8153, 1219, 565, 1, 10, 98, 4096, 0, 8, 22] payload_rows_per_dst=[16382, 16384, 16238, 16384, 16384, 15616, 16384, 15728] dup_rows_per_dst=[21318, 16126, 15456, 24810, 32066, 9472, 4038, 9358] combine_rows_per_src=[16382, 16384, 16238, 16384, 16384, 15616, 16384, 15728] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130258 payload_rows_remote=113874 dup_rows=131886 dup_rows_remote=109412 dedup_ratio=0.503105 router_tokens_per_expert=[0, 661, 8186, 7699, 1, 34, 3, 0, 28, 0, 0, 7960, 1, 0, 0, 0, 0, 23, 1, 8085, 3, 8154, 8130, 13, 7476, 38, 34, 0, 8191, 16, 8180, 8, 0, 8189, 3, 4, 7182, 28, 8083, 36, 7833, 0, 531, 0, 7869, 192, 8192, 212, 10, 0, 125, 11, 0, 0, 8179, 914, 0, 0, 0, 287, 234, 29, 4, 0] payload_rows_per_dst=[15666, 16384, 16344, 16384, 16384, 16328, 16384, 16384] dup_rows_per_dst=[1130, 15966, 32474, 15120, 22474, 17696, 10286, 16740] combine_rows_per_src=[15666, 16384, 16344, 16384, 16384, 16328, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938590208 dispatch_meta_bytes=933784 combine_payload_bytes=938590208 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=939523992 ingress_bytes=1378992128 link_bytes=1378992128 token_bytes=1878114200 weight_movement_bytes=440401920 total_nvlink_bytes=2318516120 weight_to_token_ratio=0.234492 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.721780297e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=932448 combine_payload_bytes=939327488 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=940259936 ingress_bytes=1379729408 link_bytes=1379729408 token_bytes=1879587424 weight_movement_bytes=440401920 total_nvlink_bytes=2319989344 weight_to_token_ratio=0.234308 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=29952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725908757e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936263680 dispatch_meta_bytes=872552 combine_payload_bytes=936263680 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=937136232 ingress_bytes=1376665600 link_bytes=1376665600 token_bytes=1873399912 weight_movement_bytes=440401920 total_nvlink_bytes=2313801832 weight_to_token_ratio=0.235082 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.708752711e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130990 payload_rows_remote=114606 dup_rows=131154 dup_rows_remote=114346 dedup_ratio=0.500313 router_tokens_per_expert=[2248, 8178, 3, 0, 6, 944, 0, 4216, 6954, 0, 16, 0, 79, 11, 1, 0, 5698, 0, 0, 8, 93, 8192, 0, 0, 7049, 0, 0, 8119, 5, 0, 8165, 8190, 8146, 0, 2, 8157, 99, 27, 2171, 8185, 8192, 19, 0, 7, 0, 0, 1, 1, 11, 97, 0, 0, 0, 3, 188, 8185, 6951, 8156, 0, 4299, 0, 0, 0, 0] payload_rows_per_dst=[16368, 16384, 16384, 16332, 16384, 16384, 16384, 16370] dup_rows_per_dst=[14822, 14120, 11598, 14108, 20820, 16436, 16808, 22442] combine_rows_per_src=[16368, 16384, 16384, 16332, 16384, 16384, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=926646272 dispatch_meta_bytes=918536 combine_payload_bytes=926646272 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=927564808 ingress_bytes=1190887424 link_bytes=1190887424 token_bytes=1854211080 weight_movement_bytes=264241152 total_nvlink_bytes=2118452232 weight_to_token_ratio=0.142509 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017992 gemm_active_groups=21 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.668472474e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=932855808 dispatch_meta_bytes=893144 combine_payload_bytes=932855808 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=933748952 ingress_bytes=1373257728 link_bytes=1373257728 token_bytes=1866604760 weight_movement_bytes=440401920 total_nvlink_bytes=2307006680 weight_to_token_ratio=0.235937 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=28288 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.689670051e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131018 payload_rows_remote=114664 dup_rows=131126 dup_rows_remote=115246 dedup_ratio=0.500206 router_tokens_per_expert=[357, 8187, 0, 0, 1681, 1965, 7, 301, 5798, 41, 0, 4, 109, 298, 0, 76, 6860, 6, 0, 0, 18, 8176, 5, 0, 6232, 341, 0, 8158, 8, 123, 8094, 7737, 7729, 0, 6, 8167, 1, 221, 8024, 8118, 8039, 50, 4, 0, 0, 41, 0, 99, 31, 1, 189, 14, 28, 0, 1527, 7584, 8054, 8157, 34, 354, 18, 0, 0, 0] payload_rows_per_dst=[16382, 16376, 16370, 16384, 16384, 16354, 16384, 16384] dup_rows_per_dst=[8614, 12310, 13760, 13206, 31944, 15880, 18562, 16850] combine_rows_per_src=[16382, 16376, 16370, 16384, 16384, 16354, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129786 payload_rows_remote=113454 dup_rows=132358 dup_rows_remote=122588 dedup_ratio=0.504906 router_tokens_per_expert=[3903, 8041, 159, 0, 2902, 1349, 68, 4724, 8015, 44, 37, 0, 0, 1312, 21, 0, 6322, 0, 100, 4, 462, 8172, 0, 10, 3353, 47, 0, 7762, 0, 1, 6548, 8141, 8147, 0, 31, 8143, 7, 0, 5649, 8162, 8158, 1, 5, 0, 0, 0, 11, 11, 241, 26, 89, 1, 0, 0, 9, 8133, 4487, 8120, 0, 96, 0, 0, 46, 2] payload_rows_per_dst=[16384, 16374, 16384, 15194, 16384, 16350, 16384, 16332] dup_rows_per_dst=[24300, 13416, 13756, 9360, 28578, 16324, 16854, 9770] combine_rows_per_src=[16384, 16374, 16384, 15194, 16384, 16350, 16384, 16332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938852352 dispatch_meta_bytes=915808 combine_payload_bytes=938852352 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939768160 ingress_bytes=1115013120 link_bytes=1115013120 token_bytes=1878620512 weight_movement_bytes=176160768 total_nvlink_bytes=2054781280 weight_to_token_ratio=0.093771 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243608043e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130884 payload_rows_remote=114500 dup_rows=131260 dup_rows_remote=114964 dedup_ratio=0.500717 router_tokens_per_expert=[5469, 8192, 0, 1, 54, 2, 2, 2247, 7646, 0, 0, 0, 0, 1485, 1, 0, 6215, 0, 4, 0, 0, 8191, 0, 2, 8085, 0, 0, 8187, 0, 0, 8119, 8192, 7713, 0, 0, 8192, 18, 0, 2701, 8192, 8192, 0, 0, 0, 0, 1, 3, 0, 196, 0, 34, 0, 0, 0, 43, 8192, 3259, 8192, 0, 4013, 0, 37, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16198, 16384, 16384, 16384] dup_rows_per_dst=[22394, 18208, 12442, 16296, 14206, 16392, 16704, 14618] combine_rows_per_src=[16384, 16384, 16382, 16384, 16198, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130396 payload_rows_remote=114014 dup_rows=131748 dup_rows_remote=115224 dedup_ratio=0.502579 router_tokens_per_expert=[226, 8190, 0, 0, 109, 1350, 0, 125, 8144, 0, 0, 0, 1, 25, 2, 0, 8191, 1, 39, 0, 41, 8173, 7, 1, 7744, 4, 0, 8046, 0, 7, 8191, 8065, 7697, 0, 0, 8187, 0, 5, 3472, 8185, 8153, 131, 129, 7, 0, 0, 15, 0, 0, 4, 182, 0, 0, 0, 64, 8141, 8152, 8188, 0, 3660, 0, 18, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 15710, 16384, 16384, 16384] dup_rows_per_dst=[11808, 16342, 16524, 23922, 14308, 16616, 8576, 23652] combine_rows_per_src=[16384, 16384, 16382, 16384, 15710, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=919640 combine_payload_bytes=939327488 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940247128 ingress_bytes=1115488256 link_bytes=1115488256 token_bytes=1879574616 weight_movement_bytes=176160768 total_nvlink_bytes=2055735384 weight_to_token_ratio=0.093724 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246268606e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129460 payload_rows_remote=113576 dup_rows=132684 dup_rows_remote=118726 dedup_ratio=0.506149 router_tokens_per_expert=[1271, 7819, 12, 0, 1533, 206, 0, 0, 7783, 15, 6, 1, 3396, 1632, 68, 12, 7034, 6, 0, 2, 0, 8159, 1, 19, 7508, 317, 0, 6678, 264, 8, 7617, 8133, 5798, 8, 1, 8148, 11, 25, 4096, 8176, 8153, 0, 14, 5, 3, 19, 278, 223, 56, 209, 244, 1, 63, 14, 2290, 8122, 3272, 8178, 0, 4160, 1, 0, 0, 4] payload_rows_per_dst=[15884, 16384, 16384, 16382, 16384, 16384, 16384, 15274] dup_rows_per_dst=[13958, 25046, 23548, 34360, 4854, 15552, 11772, 3594] combine_rows_per_src=[15884, 16384, 16384, 16382, 16384, 16384, 16384, 15274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=929415168 dispatch_meta_bytes=944168 combine_payload_bytes=929415168 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=930359336 ingress_bytes=1281736704 link_bytes=1281736704 token_bytes=1859774504 weight_movement_bytes=352321536 total_nvlink_bytes=2212096040 weight_to_token_ratio=0.189443 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.177190519e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124566 payload_rows_remote=108182 dup_rows=137578 dup_rows_remote=122806 dedup_ratio=0.524818 router_tokens_per_expert=[4089, 8119, 0, 0, 6, 27, 0, 4851, 7169, 0, 192, 1, 234, 0, 0, 0, 7784, 0, 687, 0, 667, 8085, 62, 0, 2, 411, 0, 7651, 0, 8, 7738, 8095, 7041, 3, 1, 8152, 175, 1, 228, 7995, 8067, 0, 1, 0, 17, 39, 0, 105, 326, 27, 0, 24, 0, 1, 4248, 8114, 8085, 8144, 79, 4250, 0, 70, 1, 0] payload_rows_per_dst=[16312, 16384, 16384, 16332, 10002, 16384, 16384, 16384] dup_rows_per_dst=[25672, 14772, 23150, 15668, 7618, 16224, 17286, 17188] combine_rows_per_src=[16312, 16384, 16384, 16332, 10002, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937984000 dispatch_meta_bytes=917856 combine_payload_bytes=937984000 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938901856 ingress_bytes=1466466304 link_bytes=1466466304 token_bytes=1876885856 weight_movement_bytes=528482304 total_nvlink_bytes=2405368160 weight_to_token_ratio=0.281574 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.211599169e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128418 payload_rows_remote=112036 dup_rows=133726 dup_rows_remote=117738 dedup_ratio=0.510124 router_tokens_per_expert=[2556, 8183, 0, 134, 353, 838, 1, 1174, 8018, 20, 22, 27, 0, 335, 0, 16, 8175, 0, 31, 1, 30, 8111, 0, 0, 6874, 1, 0, 8101, 0, 0, 4098, 8190, 7759, 2, 19, 8186, 0, 11, 254, 8142, 8170, 0, 0, 36, 1, 32, 619, 1, 36, 0, 0, 2, 0, 0, 4168, 8086, 7676, 8192, 0, 4378, 0, 8, 5, 0] payload_rows_per_dst=[16384, 16384, 16370, 13752, 16382, 16384, 16378, 16384] dup_rows_per_dst=[10094, 16876, 16326, 8080, 15988, 17712, 24516, 24134] combine_rows_per_src=[16384, 16384, 16370, 13752, 16382, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=934002688 dispatch_meta_bytes=916952 combine_payload_bytes=934002688 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=934919640 ingress_bytes=1110163456 link_bytes=1110163456 token_bytes=1868922328 weight_movement_bytes=176160768 total_nvlink_bytes=2045083096 weight_to_token_ratio=0.094258 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.216451949e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=930414592 dispatch_meta_bytes=929208 combine_payload_bytes=930414592 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=931343800 ingress_bytes=1194655744 link_bytes=1194655744 token_bytes=1861758392 weight_movement_bytes=264241152 total_nvlink_bytes=2125999544 weight_to_token_ratio=0.141931 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.689573493e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=886226944 dispatch_meta_bytes=923952 combine_payload_bytes=886226944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=887150896 ingress_bytes=1062387712 link_bytes=1062387712 token_bytes=1773377840 weight_movement_bytes=176160768 total_nvlink_bytes=1949538608 weight_to_token_ratio=0.099336 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.948927725e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130600 payload_rows_remote=114290 dup_rows=131544 dup_rows_remote=112254 dedup_ratio=0.501801 router_tokens_per_expert=[0, 8031, 0, 0, 0, 16, 5586, 5996, 0, 1797, 4044, 4, 72, 0, 0, 440, 3654, 274, 4075, 0, 33, 7579, 81, 8103, 9, 0, 0, 4188, 0, 114, 51, 92, 8176, 0, 3, 2, 1617, 7777, 7970, 0, 0, 1886, 8182, 0, 0, 0, 139, 0, 3, 0, 0, 8155, 4918, 8190, 0, 8170, 8165, 1, 6, 3344, 0, 1, 128, 0] payload_rows_per_dst=[16016, 16384, 16384, 16384, 16364, 16374, 16310, 16384] dup_rows_per_dst=[14998, 12710, 23252, 8464, 13684, 16034, 19290, 23112] combine_rows_per_src=[16016, 16384, 16384, 16384, 16364, 16374, 16310, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=917798912 dispatch_meta_bytes=919096 combine_payload_bytes=917798912 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=918718008 ingress_bytes=1534361600 link_bytes=1534361600 token_bytes=1836516920 weight_movement_bytes=616562688 total_nvlink_bytes=2453079608 weight_to_token_ratio=0.335724 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.591784486e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130702 payload_rows_remote=114318 dup_rows=131442 dup_rows_remote=114830 dedup_ratio=0.501411 router_tokens_per_expert=[90, 7544, 57, 0, 741, 17, 4351, 4994, 0, 0, 2831, 48, 3364, 0, 0, 2903, 3791, 0, 7456, 0, 0, 5321, 931, 8143, 0, 9, 23, 7642, 49, 517, 2518, 0, 0, 0, 0, 0, 7180, 6067, 7784, 27, 0, 0, 8128, 0, 215, 12, 0, 0, 0, 0, 47, 4898, 7819, 8162, 235, 8120, 8181, 0, 1, 451, 126, 0, 2, 277] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16208, 16384, 16192, 16384] dup_rows_per_dst=[19148, 17444, 18670, 21360, 10340, 16612, 9806, 18062] combine_rows_per_src=[16384, 16382, 16384, 16384, 16208, 16384, 16192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 2, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130698 payload_rows_remote=114322 dup_rows=131446 dup_rows_remote=113700 dedup_ratio=0.501427 router_tokens_per_expert=[262, 7617, 290, 0, 40, 1, 2449, 7992, 0, 174, 4209, 0, 3, 21, 833, 0, 603, 116, 7365, 2, 0, 3155, 1681, 8070, 0, 0, 58, 4103, 0, 1437, 3, 0, 7853, 59, 0, 0, 8016, 8070, 8162, 8, 2, 110, 8079, 7, 0, 37, 1, 0, 1, 3, 130, 7568, 7286, 7876, 29, 8184, 8133, 3, 0, 858, 45, 6, 62, 0] payload_rows_per_dst=[16154, 16384, 16384, 16384, 16380, 16252, 16384, 16376] dup_rows_per_dst=[5914, 10238, 24694, 11140, 23672, 8254, 29788, 17746] combine_rows_per_src=[16154, 16384, 16384, 16384, 16380, 16252, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936263680 dispatch_meta_bytes=906176 combine_payload_bytes=936263680 prefetch_bytes=88080384 reduce_bytes=704643072 egress_bytes=937169856 ingress_bytes=1728987136 link_bytes=1728987136 token_bytes=1873433536 weight_movement_bytes=792723456 total_nvlink_bytes=2666156992 weight_to_token_ratio=0.423139 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=28800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.681606247e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130328 payload_rows_remote=113944 dup_rows=131816 dup_rows_remote=115316 dedup_ratio=0.502838 router_tokens_per_expert=[0, 7670, 33, 0, 0, 0, 1696, 5283, 0, 289, 4254, 0, 2330, 0, 460, 0, 45, 0, 8090, 0, 0, 4100, 682, 8075, 0, 0, 0, 8190, 3, 51, 6, 0, 4371, 0, 0, 1, 7749, 7822, 8183, 0, 0, 2304, 8192, 0, 4, 0, 160, 19, 0, 0, 0, 7618, 8027, 8164, 0, 8192, 8161, 0, 0, 831, 0, 1, 0, 16] payload_rows_per_dst=[15702, 16378, 16372, 16384, 16370, 16384, 16384, 16354] dup_rows_per_dst=[13662, 14628, 9462, 16500, 23516, 21124, 19056, 13868] combine_rows_per_src=[15702, 16378, 16372, 16384, 16370, 16384, 16384, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936493056 dispatch_meta_bytes=916592 combine_payload_bytes=936493056 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=937409648 ingress_bytes=1112653824 link_bytes=1112653824 token_bytes=1873902704 weight_movement_bytes=176160768 total_nvlink_bytes=2050063472 weight_to_token_ratio=0.094007 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.230396970e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129016 payload_rows_remote=112634 dup_rows=133128 dup_rows_remote=116740 dedup_ratio=0.507843 router_tokens_per_expert=[6, 6880, 1, 0, 0, 0, 3897, 5211, 29, 362, 5587, 48, 23, 0, 0, 228, 156, 0, 7846, 0, 0, 6813, 30, 8184, 1, 0, 0, 8154, 798, 1845, 50, 0, 3535, 0, 3, 0, 3759, 8145, 8192, 0, 8, 0, 8177, 27, 0, 0, 1424, 0, 0, 0, 0, 8172, 7684, 8109, 4, 8150, 8192, 0, 0, 1033, 6, 48, 255, 0] payload_rows_per_dst=[14390, 16384, 16382, 16384, 16352, 16384, 16356, 16384] dup_rows_per_dst=[17600, 12538, 16388, 21696, 14532, 19188, 23406, 7780] combine_rows_per_src=[14390, 16384, 16382, 16384, 16352, 16384, 16356, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124664 payload_rows_remote=108588 dup_rows=137480 dup_rows_remote=118074 dedup_ratio=0.524445 router_tokens_per_expert=[2, 4037, 361, 0, 0, 0, 3820, 7813, 23, 8175, 6249, 8, 0, 10, 0, 18, 5, 766, 3417, 0, 0, 7241, 122, 7529, 0, 23, 0, 2601, 1606, 4203, 1146, 59, 5050, 3, 0, 0, 4096, 7307, 8164, 0, 1, 0, 4562, 788, 0, 38, 279, 2, 16, 0, 2258, 7013, 4097, 8084, 0, 7895, 8139, 13, 22, 3862, 3, 0, 142, 4] payload_rows_per_dst=[16076, 16384, 16324, 16384, 15466, 11706, 15972, 16352] dup_rows_per_dst=[19406, 35752, 18370, 30240, 13830, 10706, 1158, 8018] combine_rows_per_src=[16076, 16384, 16324, 16384, 15466, 11706, 15972, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936525824 dispatch_meta_bytes=912088 combine_payload_bytes=936525824 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=937437912 ingress_bytes=1112686592 link_bytes=1112686592 token_bytes=1873963736 weight_movement_bytes=176160768 total_nvlink_bytes=2050124504 weight_to_token_ratio=0.094004 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.230580457e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129608 payload_rows_remote=113408 dup_rows=132536 dup_rows_remote=116516 dedup_ratio=0.505585 router_tokens_per_expert=[0, 8189, 420, 0, 18, 0, 4103, 8100, 0, 11, 4092, 0, 0, 0, 0, 3912, 1562, 61, 240, 7, 0, 8175, 672, 8170, 0, 31, 0, 4794, 1, 213, 227, 105, 6283, 39, 251, 0, 119, 8117, 7617, 8, 0, 7018, 5502, 0, 51, 8, 1550, 225, 1, 0, 76, 7897, 4054, 7742, 48, 8191, 8132, 12, 1, 3505, 0, 1044, 0, 478] payload_rows_per_dst=[16384, 16200, 16378, 15512, 16384, 16384, 15982, 16384] dup_rows_per_dst=[25276, 16020, 13244, 12616, 13250, 28666, 13504, 9960] combine_rows_per_src=[16384, 16200, 16378, 15512, 16384, 16384, 15982, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=933429248 dispatch_meta_bytes=917040 combine_payload_bytes=933429248 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934346288 ingress_bytes=1285750784 link_bytes=1285750784 token_bytes=1867775536 weight_movement_bytes=352321536 total_nvlink_bytes=2220097072 weight_to_token_ratio=0.188632 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.199667692e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130736 payload_rows_remote=114354 dup_rows=131408 dup_rows_remote=111994 dedup_ratio=0.501282 router_tokens_per_expert=[0, 6349, 31, 48, 78, 0, 6835, 4176, 0, 1352, 4009, 22, 830, 22, 12, 0, 308, 0, 7193, 0, 0, 4227, 0, 8016, 0, 1, 0, 4238, 54, 648, 0, 26, 8190, 0, 39, 0, 1544, 8125, 7969, 0, 8, 79, 7404, 1, 0, 0, 11, 162, 6, 7, 30, 8066, 8169, 8107, 0, 8181, 8170, 1, 0, 8039, 10, 3, 234, 42] payload_rows_per_dst=[16336, 16260, 16222, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[18698, 12058, 7234, 9928, 19414, 15308, 16172, 32596] combine_rows_per_src=[16336, 16260, 16222, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=922697728 dispatch_meta_bytes=917496 combine_payload_bytes=922697728 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=923615224 ingress_bytes=1275019264 link_bytes=1275019264 token_bytes=1846312952 weight_movement_bytes=352321536 total_nvlink_bytes=2198634488 weight_to_token_ratio=0.190824 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=24704 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.139575659e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=889552896 dispatch_meta_bytes=906648 combine_payload_bytes=889552896 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=890459544 ingress_bytes=1241874432 link_bytes=1241874432 token_bytes=1780012440 weight_movement_bytes=352321536 total_nvlink_bytes=2132333976 weight_to_token_ratio=0.197932 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=25600 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.953978435e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 2, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128612 payload_rows_remote=112228 dup_rows=133532 dup_rows_remote=111428 dedup_ratio=0.509384 router_tokens_per_expert=[0, 0, 8134, 94, 4, 0, 8191, 0, 0, 57, 1, 0, 0, 37, 124, 566, 5449, 8192, 1, 0, 7, 6659, 8126, 94, 7009, 17, 0, 1, 14, 8151, 226, 8058, 8188, 0, 110, 8172, 0, 8180, 56, 0, 0, 1, 28, 172, 238, 557, 0, 8192, 11, 4, 0, 0, 0, 0, 0, 3140, 1373, 6100, 1152, 66, 8190, 34, 7896, 0] payload_rows_per_dst=[16384, 16384, 16298, 14076, 16384, 16384, 16384, 16318] dup_rows_per_dst=[196, 11198, 31012, 458, 29094, 10102, 22104, 29368] combine_rows_per_src=[16384, 16384, 16298, 14076, 16384, 16384, 16384, 16318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=929038336 dispatch_meta_bytes=919696 combine_payload_bytes=929038336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=929958032 ingress_bytes=1281359872 link_bytes=1281359872 token_bytes=1858996368 weight_movement_bytes=352321536 total_nvlink_bytes=2211317904 weight_to_token_ratio=0.189522 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.019886 gemm_active_groups=22 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.175080417e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124716 payload_rows_remote=109470 dup_rows=137428 dup_rows_remote=130324 dedup_ratio=0.524246 router_tokens_per_expert=[3675, 37, 8032, 4280, 843, 0, 7814, 0, 0, 3784, 0, 0, 96, 0, 321, 0, 4101, 8059, 0, 0, 1969, 4176, 4095, 118, 4053, 2015, 303, 6, 0, 7594, 579, 6834, 7974, 7, 0, 4360, 15, 7106, 51, 0, 1, 116, 23, 7, 119, 2, 0, 7383, 1, 0, 12, 0, 0, 10, 0, 1978, 4102, 4527, 0, 4078, 8191, 119, 8041, 65] payload_rows_per_dst=[16354, 16384, 16384, 11224, 16384, 15246, 16356, 16384] dup_rows_per_dst=[16826, 16326, 20526, 5712, 22642, 7104, 17526, 30766] combine_rows_per_src=[16354, 16384, 16384, 11224, 16384, 15246, 16356, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936787968 dispatch_meta_bytes=905392 combine_payload_bytes=936787968 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=937693360 ingress_bytes=1553350656 link_bytes=1553350656 token_bytes=1874481328 weight_movement_bytes=616562688 total_nvlink_bytes=2491044016 weight_to_token_ratio=0.328924 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=28800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.698115273e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129412 payload_rows_remote=113292 dup_rows=132732 dup_rows_remote=94368 dedup_ratio=0.506332 router_tokens_per_expert=[4969, 0, 8177, 36, 315, 0, 8151, 4, 62, 0, 117, 0, 403, 0, 2, 2, 4915, 8069, 1, 0, 209, 371, 7814, 2684, 7624, 3002, 1731, 5, 8, 8169, 4, 7893, 3935, 21, 1, 6487, 0, 8166, 0, 3, 0, 136, 0, 7, 4, 4089, 44, 8052, 0, 0, 0, 38, 7, 0, 136, 762, 4440, 3594, 116, 6, 8145, 0, 8141, 5] payload_rows_per_dst=[15230, 16380, 16384, 16152, 16384, 16380, 16382, 16120] dup_rows_per_dst=[10308, 930, 31974, 8596, 14046, 16476, 12038, 38364] combine_rows_per_src=[15230, 16380, 16384, 16152, 16384, 16380, 16382, 16120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=919371776 dispatch_meta_bytes=894624 combine_payload_bytes=919371776 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=920266400 ingress_bytes=1271693312 link_bytes=1271693312 token_bytes=1839638176 weight_movement_bytes=352321536 total_nvlink_bytes=2191959712 weight_to_token_ratio=0.191517 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.018939 gemm_active_groups=22 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.120951716e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=896778240 dispatch_meta_bytes=959176 combine_payload_bytes=896778240 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=897737416 ingress_bytes=1072939008 link_bytes=1072939008 token_bytes=1794515656 weight_movement_bytes=176160768 total_nvlink_bytes=1970676424 weight_to_token_ratio=0.098166 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.008010578e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130804 payload_rows_remote=114516 dup_rows=131340 dup_rows_remote=113228 dedup_ratio=0.501022 router_tokens_per_expert=[839, 0, 8192, 2784, 4, 3, 8187, 0, 0, 156, 9, 0, 0, 0, 1, 0, 4326, 8179, 692, 0, 4821, 484, 4087, 4092, 7606, 14, 3621, 0, 0, 8145, 0, 7332, 182, 0, 0, 7834, 0, 7924, 0, 0, 0, 250, 0, 2997, 207, 4539, 0, 8186, 0, 34, 0, 0, 0, 1, 0, 560, 3685, 4486, 239, 0, 8185, 0, 8189, 0] payload_rows_per_dst=[16384, 16382, 16384, 16288, 16382, 16384, 16354, 16246] dup_rows_per_dst=[23610, 8482, 12470, 18112, 15498, 30996, 13894, 8278] combine_rows_per_src=[16384, 16382, 16384, 16288, 16382, 16384, 16354, 16246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129708 payload_rows_remote=113328 dup_rows=132436 dup_rows_remote=120526 dedup_ratio=0.505203 router_tokens_per_expert=[3877, 0, 8192, 6, 0, 1, 8160, 5, 1, 92, 0, 0, 1, 24, 49, 0, 4047, 8141, 0, 115, 85, 5935, 7584, 3963, 8129, 265, 1424, 0, 0, 8185, 226, 8191, 0, 0, 1, 4029, 0, 8169, 0, 0, 0, 5, 0, 3463, 93, 159, 0, 8148, 0, 21, 6, 1, 0, 39, 0, 3532, 3391, 4198, 2367, 339, 8190, 0, 8186, 37] payload_rows_per_dst=[16384, 16378, 16380, 16384, 16338, 16342, 16384, 15118] dup_rows_per_dst=[24098, 15406, 11910, 36452, 8060, 15586, 15378, 5546] combine_rows_per_src=[16384, 16378, 16380, 16384, 16338, 16342, 16384, 15118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=118660 payload_rows_remote=102276 dup_rows=143484 dup_rows_remote=119668 dedup_ratio=0.547348 router_tokens_per_expert=[813, 1, 8178, 45, 2739, 121, 7922, 0, 0, 2306, 1458, 28, 4330, 0, 3980, 1166, 2301, 8177, 1553, 0, 231, 84, 2537, 17, 7546, 81, 927, 0, 12, 7936, 1, 6796, 4053, 0, 387, 7805, 3, 6865, 90, 91, 1, 6, 25, 3969, 264, 350, 5, 5388, 18, 0, 0, 2, 0, 1213, 16, 3427, 6073, 7601, 66, 1, 5119, 76, 6872, 0] payload_rows_per_dst=[16384, 16384, 8210, 16332, 16384, 12876, 16092, 15998] dup_rows_per_dst=[23816, 31580, 162, 35368, 16664, 15292, 8966, 11636] combine_rows_per_src=[16384, 16384, 8210, 16332, 16384, 12876, 16092, 15998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=928088064 dispatch_meta_bytes=830640 combine_payload_bytes=928088064 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=928918704 ingress_bytes=1456570368 link_bytes=1456570368 token_bytes=1857006768 weight_movement_bytes=528482304 total_nvlink_bytes=2385489072 weight_to_token_ratio=0.284588 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=23552 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.156186058e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125366 payload_rows_remote=108982 dup_rows=136778 dup_rows_remote=115422 dedup_ratio=0.521767 router_tokens_per_expert=[1091, 251, 7881, 3041, 0, 4, 7848, 63, 2, 3806, 0, 1, 0, 102, 41, 240, 4789, 8189, 30, 308, 3780, 3188, 6489, 76, 5390, 3, 14, 0, 0, 7726, 125, 8080, 1141, 0, 0, 7227, 91, 7932, 59, 5, 3, 17, 0, 3749, 0, 890, 0, 8078, 4, 140, 0, 52, 7, 28, 40, 3426, 5580, 582, 551, 3159, 7749, 1, 8003, 0] payload_rows_per_dst=[16384, 16384, 15948, 16384, 16296, 16360, 15750, 11860] dup_rows_per_dst=[23974, 21356, 8394, 28734, 14228, 16954, 15252, 7886] combine_rows_per_src=[16384, 16384, 15948, 16384, 16296, 16360, 15750, 11860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938115072 dispatch_meta_bytes=910976 combine_payload_bytes=938115072 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939026048 ingress_bytes=1466597376 link_bytes=1466597376 token_bytes=1877141120 weight_movement_bytes=528482304 total_nvlink_bytes=2405623424 weight_to_token_ratio=0.281536 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.212333118e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129524 payload_rows_remote=113172 dup_rows=132620 dup_rows_remote=121612 dedup_ratio=0.505905 router_tokens_per_expert=[539, 0, 8186, 24, 1513, 20, 8138, 13, 0, 5796, 3, 4, 158, 421, 0, 10, 3502, 8181, 3460, 0, 164, 85, 6522, 7116, 7317, 72, 93, 5, 0, 8076, 4, 7536, 21, 15, 18, 5455, 30, 8141, 0, 0, 0, 378, 1, 81, 0, 4102, 0, 7553, 1, 0, 32, 0, 44, 0, 0, 5222, 1948, 4092, 1, 690, 8090, 9, 8182, 8] payload_rows_per_dst=[16382, 16384, 16374, 14942, 16352, 16384, 16384, 16322] dup_rows_per_dst=[9430, 25806, 23334, 6930, 11008, 23278, 26900, 5934] combine_rows_per_src=[16382, 16384, 16374, 14942, 16352, 16384, 16384, 16322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=928382976 dispatch_meta_bytes=935416 combine_payload_bytes=928382976 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=929318392 ingress_bytes=1456865280 link_bytes=1456865280 token_bytes=1857701368 weight_movement_bytes=528482304 total_nvlink_bytes=2386183672 weight_to_token_ratio=0.284482 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.023674 gemm_active_groups=18 gemm_peak_group_rows=17152 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.157837443e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=837844992 dispatch_meta_bytes=887776 combine_payload_bytes=837844992 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=838732768 ingress_bytes=1190166528 link_bytes=1190166528 token_bytes=1676577760 weight_movement_bytes=352321536 total_nvlink_bytes=2028899296 weight_to_token_ratio=0.210143 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.664435757e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130862 payload_rows_remote=114478 dup_rows=131282 dup_rows_remote=99532 dedup_ratio=0.500801 router_tokens_per_expert=[0, 8169, 107, 43, 0, 5242, 0, 0, 1, 3239, 22, 8133, 8180, 8191, 65, 0, 0, 34, 32, 0, 12, 0, 880, 36, 0, 0, 12, 73, 0, 12, 0, 39, 8106, 3073, 7996, 0, 343, 0, 0, 70, 7589, 5969, 212, 5888, 21, 12, 0, 0, 8163, 287, 0, 8032, 8067, 7, 8106, 2, 8125, 4, 8, 0, 0, 8159, 303, 8] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16358, 16254, 16384, 16334] dup_rows_per_dst=[10742, 6572, 9822, 16624, 22818, 16074, 31750, 16880] combine_rows_per_src=[16380, 16384, 16384, 16384, 16358, 16254, 16384, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=892780544 dispatch_meta_bytes=897616 combine_payload_bytes=892780544 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=893678160 ingress_bytes=1245102080 link_bytes=1245102080 token_bytes=1786458704 weight_movement_bytes=352321536 total_nvlink_bytes=2138780240 weight_to_token_ratio=0.197218 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.972051917e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=927105024 dispatch_meta_bytes=939136 combine_payload_bytes=927105024 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=928044160 ingress_bytes=1279426560 link_bytes=1279426560 token_bytes=1855149184 weight_movement_bytes=352321536 total_nvlink_bytes=2207470720 weight_to_token_ratio=0.189915 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=27648 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.164254677e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130502 payload_rows_remote=114136 dup_rows=131642 dup_rows_remote=108604 dedup_ratio=0.502174 router_tokens_per_expert=[66, 8090, 2211, 0, 0, 5939, 134, 1, 0, 3301, 76, 7940, 8169, 8021, 0, 27, 0, 2, 0, 0, 241, 0, 248, 0, 0, 1, 0, 0, 0, 0, 43, 0, 6719, 4221, 7972, 317, 884, 0, 0, 132, 8080, 7187, 695, 5880, 0, 0, 0, 511, 5430, 0, 17, 7537, 8086, 15, 5248, 51, 8179, 1137, 4, 34, 0, 8173, 53, 0] payload_rows_per_dst=[16384, 16384, 16188, 16362, 16384, 16366, 16050, 16384] dup_rows_per_dst=[16498, 6680, 13798, 14698, 24106, 23038, 14048, 18776] combine_rows_per_src=[16384, 16384, 16188, 16362, 16384, 16366, 16050, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130460 payload_rows_remote=114076 dup_rows=131684 dup_rows_remote=106520 dedup_ratio=0.502335 router_tokens_per_expert=[1, 8190, 1322, 86, 11, 3085, 14, 4, 7, 6606, 0, 8164, 8186, 8088, 2680, 25, 269, 0, 103, 0, 0, 0, 275, 8, 1, 8, 107, 0, 23, 79, 24, 0, 7893, 120, 8170, 0, 1495, 42, 0, 1, 726, 8128, 241, 7894, 16, 0, 0, 0, 7415, 10, 0, 8135, 8153, 1, 7251, 0, 8144, 530, 3, 1162, 1, 8173, 0, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 15804, 16352, 16384] dup_rows_per_dst=[9042, 18598, 9360, 15162, 13540, 1950, 38868, 25164] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 15804, 16352, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 2, 3, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=937803776 dispatch_meta_bytes=856040 combine_payload_bytes=937803776 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=938659816 ingress_bytes=1554366464 link_bytes=1554366464 token_bytes=1876463592 weight_movement_bytes=616562688 total_nvlink_bytes=2493026280 weight_to_token_ratio=0.328577 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=28032 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.703803374e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129658 payload_rows_remote=113960 dup_rows=132486 dup_rows_remote=117458 dedup_ratio=0.505394 router_tokens_per_expert=[0, 7804, 3074, 1, 0, 164, 0, 0, 532, 8187, 0, 8190, 8191, 8189, 24, 138, 2026, 0, 0, 0, 17, 0, 2108, 0, 0, 0, 89, 0, 0, 0, 0, 0, 3099, 26, 8182, 0, 271, 0, 0, 0, 5694, 8102, 5, 7830, 0, 0, 0, 211, 7845, 0, 0, 8185, 7428, 0, 7999, 0, 8192, 0, 495, 68, 0, 8190, 516, 0] payload_rows_per_dst=[15656, 16384, 16384, 15698, 16384, 16384, 16384, 16384] dup_rows_per_dst=[6430, 20998, 24668, 15028, 6772, 27300, 15996, 15294] combine_rows_per_src=[15656, 16384, 16384, 15698, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=113440 dedup_ratio=0.500084 router_tokens_per_expert=[0, 8178, 1281, 0, 0, 7195, 0, 3, 114, 4096, 0, 8026, 8185, 8183, 247, 33, 0, 0, 0, 0, 0, 1, 3436, 0, 0, 1, 200, 12, 0, 0, 0, 0, 8170, 367, 8059, 0, 26, 0, 0, 0, 648, 7880, 907, 7899, 1, 7, 0, 0, 8122, 15, 0, 8192, 7528, 0, 6195, 216, 8168, 0, 149, 46, 1, 8191, 1088, 6] payload_rows_per_dst=[16362, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16952, 21114, 17654, 16776, 16860, 18300, 12714, 10724] combine_rows_per_src=[16362, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=935002112 dispatch_meta_bytes=890960 combine_payload_bytes=935002112 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=935893072 ingress_bytes=1287323648 link_bytes=1287323648 token_bytes=1870895184 weight_movement_bytes=352321536 total_nvlink_bytes=2223216720 weight_to_token_ratio=0.188317 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=25600 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.208475073e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934510592 dispatch_meta_bytes=882384 combine_payload_bytes=934510592 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=935392976 ingress_bytes=1374912512 link_bytes=1374912512 token_bytes=1869903568 weight_movement_bytes=440401920 total_nvlink_bytes=2310305488 weight_to_token_ratio=0.235521 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.698936150e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=119636 payload_rows_remote=103252 dup_rows=142508 dup_rows_remote=109912 dedup_ratio=0.543625 router_tokens_per_expert=[166, 4106, 3991, 0, 3, 8192, 0, 0, 953, 21, 0, 7011, 8146, 8182, 0, 1999, 0, 1, 11, 12, 875, 0, 271, 3559, 93, 0, 889, 0, 3, 0, 45, 12, 8190, 4080, 8032, 0, 3842, 1, 472, 1, 7372, 4095, 4, 4953, 12, 38, 0, 9, 4126, 0, 0, 7540, 5298, 1, 4167, 270, 7994, 1, 219, 3599, 12, 8185, 18, 0] payload_rows_per_dst=[16384, 16368, 16384, 16384, 16384, 16384, 8946, 12402] dup_rows_per_dst=[32596, 23594, 11726, 20538, 18204, 22460, 3780, 9610] combine_rows_per_src=[16384, 16368, 16384, 16384, 16384, 16384, 8946, 12402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=10 unused_prefetch_slots=22 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=5 remote_demand_peak=3 slot_utilization=0.312500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 6, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128158 payload_rows_remote=111774 dup_rows=133986 dup_rows_remote=105262 dedup_ratio=0.511116 router_tokens_per_expert=[0, 8176, 76, 0, 30, 4452, 2, 3, 10, 6164, 1, 8040, 8156, 7213, 39, 1026, 590, 17, 19, 0, 7, 0, 123, 0, 0, 0, 14, 0, 30, 0, 0, 30, 7748, 98, 8189, 65, 4202, 0, 0, 7, 6448, 4099, 2, 4232, 0, 1, 2, 1158, 6979, 4, 0, 5400, 8072, 0, 7196, 18, 8151, 215, 3662, 2659, 8, 8142, 97, 0] payload_rows_per_dst=[16384, 16384, 16366, 16266, 16384, 15556, 14434, 16384] dup_rows_per_dst=[23652, 28724, 9350, 16124, 12132, 13872, 8838, 21294] combine_rows_per_src=[16384, 16384, 16366, 16266, 16384, 15556, 14434, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=114434 dedup_ratio=0.500107 router_tokens_per_expert=[4, 8185, 3370, 0, 0, 4529, 0, 61, 0, 574, 0, 7570, 8155, 8180, 2251, 0, 12, 4, 10, 2, 0, 0, 8042, 1746, 14, 33, 228, 59, 7, 6, 0, 0, 8182, 0, 8102, 68, 37, 0, 136, 0, 88, 4378, 333, 8187, 1, 5, 0, 17, 8086, 4, 13, 7146, 8053, 24, 7874, 423, 8112, 0, 134, 465, 2, 8159, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16358] dup_rows_per_dst=[15914, 4808, 34908, 16948, 16666, 9634, 14834, 17388] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=912424 combine_payload_bytes=939343872 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940256296 ingress_bytes=1291665408 link_bytes=1291665408 token_bytes=1879600168 weight_movement_bytes=352321536 total_nvlink_bytes=2231921704 weight_to_token_ratio=0.187445 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.018939 gemm_active_groups=21 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232787117e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=933560320 dispatch_meta_bytes=925672 combine_payload_bytes=933560320 prefetch_bytes=440401920 reduce_bytes=0 egress_bytes=934485992 ingress_bytes=1373962240 link_bytes=1373962240 token_bytes=1868046312 weight_movement_bytes=440401920 total_nvlink_bytes=2308448232 weight_to_token_ratio=0.235755 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.020833 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.693615024e-05 prefetch_bytes_per_gemm_flop=2.466066919e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=845840384 dispatch_meta_bytes=852656 combine_payload_bytes=845840384 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=846693040 ingress_bytes=1022001152 link_bytes=1022001152 token_bytes=1692533424 weight_movement_bytes=176160768 total_nvlink_bytes=1868694192 weight_to_token_ratio=0.104081 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.722779847e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=915652608 dispatch_meta_bytes=868144 combine_payload_bytes=915652608 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=916520752 ingress_bytes=1444134912 link_bytes=1444134912 token_bytes=1832173360 weight_movement_bytes=528482304 total_nvlink_bytes=2360655664 weight_to_token_ratio=0.288446 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.023674 gemm_active_groups=17 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.086552696e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=916376 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940211096 ingress_bytes=1379696640 link_bytes=1379696640 token_bytes=1879505816 weight_movement_bytes=440401920 total_nvlink_bytes=2319907736 weight_to_token_ratio=0.234318 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=29312 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725725270e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130866 payload_rows_remote=114524 dup_rows=131278 dup_rows_remote=123020 dedup_ratio=0.500786 router_tokens_per_expert=[8055, 0, 0, 0, 8168, 3156, 8191, 159, 8155, 0, 0, 0, 0, 8, 100, 0, 66, 7635, 254, 4339, 0, 8099, 7726, 3, 399, 12, 0, 0, 39, 0, 20, 0, 0, 8139, 0, 8082, 0, 0, 0, 0, 726, 5611, 0, 0, 1, 12, 7, 0, 0, 23, 28, 8154, 10, 8184, 2, 8179, 155, 7912, 3577, 0, 2, 778, 0, 6906] payload_rows_per_dst=[16336, 16384, 16384, 16384, 16332, 16320, 16342, 16384] dup_rows_per_dst=[14560, 16524, 23632, 9116, 16110, 20802, 8258, 22276] combine_rows_per_src=[16336, 16384, 16384, 16384, 16332, 16320, 16342, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130558 payload_rows_remote=114174 dup_rows=131586 dup_rows_remote=111176 dedup_ratio=0.501961 router_tokens_per_expert=[3752, 0, 1, 0, 4897, 6307, 8159, 1095, 8072, 0, 33, 7, 0, 0, 168, 385, 3, 5314, 82, 1811, 35, 8178, 6743, 0, 1480, 17, 2, 0, 1, 0, 12, 2, 0, 8152, 0, 7464, 0, 22, 114, 0, 1427, 4743, 25, 5, 300, 0, 0, 0, 458, 4357, 1, 7912, 0, 8185, 4131, 7765, 0, 8104, 3, 196, 163, 2811, 0, 8178] payload_rows_per_dst=[15936, 16384, 16384, 16384, 16318, 16384, 16384, 16384] dup_rows_per_dst=[8758, 16944, 11884, 12430, 15186, 20410, 23448, 22526] combine_rows_per_src=[15936, 16384, 16384, 16384, 16318, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129506 payload_rows_remote=113122 dup_rows=132638 dup_rows_remote=109542 dedup_ratio=0.505974 router_tokens_per_expert=[6501, 21, 0, 0, 6867, 4346, 8106, 0, 8150, 87, 0, 0, 2, 7, 0, 63, 53, 4453, 60, 875, 14, 8102, 8143, 7, 1296, 1537, 4, 10, 2, 0, 0, 0, 40, 8192, 2, 8138, 13, 1, 0, 0, 3860, 988, 0, 2, 0, 3, 167, 49, 0, 2849, 7, 8170, 0, 8163, 3907, 8075, 13, 8119, 720, 19, 34, 2741, 97, 7997] payload_rows_per_dst=[14952, 16384, 16254, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[12064, 15696, 11420, 8192, 16388, 18698, 27084, 23096] combine_rows_per_src=[14952, 16384, 16254, 16384, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 4, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130220 payload_rows_remote=113836 dup_rows=131924 dup_rows_remote=114824 dedup_ratio=0.503250 router_tokens_per_expert=[5303, 0, 0, 3, 7809, 2626, 8178, 152, 8187, 0, 0, 18, 0, 0, 0, 0, 0, 7494, 0, 84, 0, 8152, 8158, 0, 346, 1, 0, 0, 13, 0, 0, 0, 0, 8165, 50, 8077, 3, 0, 23, 1, 4169, 5452, 5, 0, 1, 0, 0, 0, 57, 298, 75, 8133, 4, 8171, 3916, 8176, 5, 7906, 3679, 0, 4, 547, 16, 7615] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 15534] dup_rows_per_dst=[15404, 16370, 17100, 17030, 18588, 19254, 16482, 11696] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 15534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130964 payload_rows_remote=114580 dup_rows=131180 dup_rows_remote=114472 dedup_ratio=0.500412 router_tokens_per_expert=[1493, 0, 0, 0, 6421, 6560, 8192, 635, 8184, 0, 0, 0, 0, 0, 0, 0, 5, 8102, 0, 4103, 0, 5644, 7691, 0, 13, 897, 0, 0, 0, 0, 93, 0, 0, 4105, 0, 8188, 0, 0, 0, 0, 3382, 8137, 0, 1, 42, 0, 0, 0, 19, 388, 9, 8137, 0, 8192, 2018, 7351, 0, 7979, 3808, 0, 0, 3091, 0, 8192] payload_rows_per_dst=[16380, 16384, 16344, 16384, 16384, 16322, 16382, 16384] dup_rows_per_dst=[13838, 16368, 18702, 16708, 8202, 22846, 6620, 27896] combine_rows_per_src=[16380, 16384, 16344, 16384, 16384, 16322, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938180608 dispatch_meta_bytes=950176 combine_payload_bytes=938180608 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939130784 ingress_bytes=1466662912 link_bytes=1466662912 token_bytes=1877311392 weight_movement_bytes=528482304 total_nvlink_bytes=2405793696 weight_to_token_ratio=0.281510 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.212700092e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=935313408 dispatch_meta_bytes=901400 combine_payload_bytes=935313408 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936214808 ingress_bytes=1287634944 link_bytes=1287634944 token_bytes=1871528216 weight_movement_bytes=352321536 total_nvlink_bytes=2223849752 weight_to_token_ratio=0.188253 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.018939 gemm_active_groups=21 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.210218201e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=926695424 dispatch_meta_bytes=890656 combine_payload_bytes=926695424 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=927586080 ingress_bytes=1631338496 link_bytes=1631338496 token_bytes=1854281504 weight_movement_bytes=704643072 total_nvlink_bytes=2558924576 weight_to_token_ratio=0.380009 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.023674 gemm_active_groups=17 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.134814623e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127522 payload_rows_remote=111144 dup_rows=134622 dup_rows_remote=112060 dedup_ratio=0.513542 router_tokens_per_expert=[4089, 270, 0, 2, 6806, 92, 7850, 4212, 8184, 1, 0, 0, 0, 2, 0, 336, 2, 3748, 0, 0, 442, 7256, 8070, 0, 3531, 492, 25, 24, 0, 3, 3903, 0, 0, 8142, 5, 7340, 1, 0, 4, 443, 4080, 434, 33, 0, 6, 1, 0, 0, 442, 1460, 0, 7058, 2, 8189, 7208, 8078, 10, 8036, 66, 36, 35, 6533, 0, 4090] payload_rows_per_dst=[16378, 16384, 16122, 16384, 16348, 16384, 16380, 13142] dup_rows_per_dst=[22562, 16428, 10176, 32106, 19980, 9042, 24110, 218] combine_rows_per_src=[16378, 16384, 16122, 16384, 16348, 16384, 16380, 13142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122438 payload_rows_remote=106054 dup_rows=139706 dup_rows_remote=123088 dedup_ratio=0.532936 router_tokens_per_expert=[4065, 0, 159, 2, 4060, 5902, 7813, 71, 8140, 0, 109, 1, 0, 30, 0, 31, 71, 5477, 98, 1659, 0, 8106, 8126, 0, 0, 6, 23, 6, 338, 2, 7, 38, 0, 7774, 39, 8067, 0, 10, 0, 0, 4622, 4089, 0, 625, 0, 9, 0, 1, 5, 2962, 1920, 3023, 19, 8086, 4627, 7646, 9, 8187, 3621, 4, 86, 3401, 9, 7891] payload_rows_per_dst=[14528, 16384, 16382, 16308, 16368, 16378, 16332, 9758] dup_rows_per_dst=[19216, 16618, 17324, 15996, 21900, 17884, 24256, 6512] combine_rows_per_src=[14528, 16384, 16382, 16308, 16368, 16378, 16332, 9758] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=932544512 dispatch_meta_bytes=914640 combine_payload_bytes=932544512 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=933459152 ingress_bytes=1372946432 link_bytes=1372946432 token_bytes=1866003664 weight_movement_bytes=440401920 total_nvlink_bytes=2306405584 weight_to_token_ratio=0.236013 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.024621 gemm_active_groups=17 gemm_peak_group_rows=30080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.687926923e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130984 payload_rows_remote=114674 dup_rows=131160 dup_rows_remote=116198 dedup_ratio=0.500336 router_tokens_per_expert=[6401, 31, 12, 41, 7985, 4315, 8132, 3640, 8152, 6, 0, 0, 31, 0, 26, 46, 82, 7628, 0, 2996, 0, 7979, 7997, 3, 1035, 612, 1, 15, 69, 82, 182, 5, 3, 7473, 14, 8085, 0, 19, 4, 38, 921, 150, 28, 228, 0, 0, 1, 0, 0, 18, 4, 8185, 149, 7800, 295, 8176, 0, 8038, 4459, 0, 318, 1633, 0, 7529] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16310, 16374, 16384, 16384] dup_rows_per_dst=[28466, 16124, 21272, 19570, 14962, 2278, 918, 27570] combine_rows_per_src=[16384, 16384, 16380, 16384, 16310, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938639360 dispatch_meta_bytes=916208 combine_payload_bytes=938639360 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939555568 ingress_bytes=1290960896 link_bytes=1290960896 token_bytes=1878194928 weight_movement_bytes=352321536 total_nvlink_bytes=2230516464 weight_to_token_ratio=0.187585 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.019886 gemm_active_groups=22 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228842144e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=910491648 dispatch_meta_bytes=892816 combine_payload_bytes=910491648 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=911384464 ingress_bytes=1438973952 link_bytes=1438973952 token_bytes=1821876112 weight_movement_bytes=528482304 total_nvlink_bytes=2350358416 weight_to_token_ratio=0.290076 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=27392 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.057653474e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=868794368 dispatch_meta_bytes=916568 combine_payload_bytes=868794368 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=869710936 ingress_bytes=1133035520 link_bytes=1133035520 token_bytes=1738505304 weight_movement_bytes=264241152 total_nvlink_bytes=2002746456 weight_to_token_ratio=0.151993 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.344525960e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=923488 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940332896 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879742304 weight_movement_bytes=176160768 total_nvlink_bytes=2055903072 weight_to_token_ratio=0.093715 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130986 payload_rows_remote=114602 dup_rows=131158 dup_rows_remote=114668 dedup_ratio=0.500328 router_tokens_per_expert=[0, 79, 0, 44, 7344, 168, 4453, 8080, 888, 1, 8133, 41, 0, 0, 17, 98, 3472, 0, 1, 4182, 7245, 3933, 0, 7836, 0, 10, 0, 10, 0, 78, 75, 8183, 0, 8162, 1, 1, 1, 0, 1, 1341, 0, 8148, 3, 942, 8178, 1, 0, 8191, 8191, 0, 4143, 3040, 5, 8055, 0, 19, 8172, 23, 63, 0, 0, 20, 0, 0] payload_rows_per_dst=[16384, 16354, 16384, 16384, 16328, 16384, 16384, 16384] dup_rows_per_dst=[11634, 10194, 23058, 16710, 18358, 18186, 16490, 16528] combine_rows_per_src=[16384, 16354, 16384, 16384, 16328, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130510 payload_rows_remote=114444 dup_rows=131634 dup_rows_remote=116248 dedup_ratio=0.502144 router_tokens_per_expert=[0, 0, 0, 5, 6894, 74, 4098, 7964, 4035, 384, 7580, 0, 5, 0, 1264, 4, 1, 0, 0, 8171, 6106, 359, 0, 8151, 46, 0, 146, 2954, 0, 118, 20, 7775, 0, 8171, 0, 0, 0, 1023, 0, 0, 72, 7680, 236, 0, 8077, 25, 39, 7674, 8192, 0, 38, 6720, 0, 8056, 0, 0, 8019, 224, 0, 56, 0, 601, 15, 0] payload_rows_per_dst=[16384, 16332, 16342, 16384, 16384, 16066, 16376, 16242] dup_rows_per_dst=[14146, 17752, 12932, 22114, 18306, 15386, 13256, 17742] combine_rows_per_src=[16384, 16332, 16342, 16384, 16384, 16066, 16376, 16242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127528 payload_rows_remote=111146 dup_rows=134616 dup_rows_remote=118210 dedup_ratio=0.513519 router_tokens_per_expert=[0, 56, 214, 0, 8100, 0, 1500, 8019, 8088, 0, 4087, 0, 8, 0, 127, 3922, 101, 0, 0, 4596, 6575, 1092, 13, 7928, 0, 0, 9, 41, 11, 0, 0, 8172, 88, 8114, 2, 15, 0, 0, 0, 49, 186, 8176, 104, 0, 8163, 0, 25, 7979, 7977, 3, 3085, 8032, 0, 8134, 0, 50, 8169, 1, 0, 21, 1, 38, 1, 0] payload_rows_per_dst=[12870, 16384, 16384, 16374, 16368, 16382, 16384, 16382] dup_rows_per_dst=[776, 23926, 8370, 7328, 16024, 22652, 39134, 16406] combine_rows_per_src=[12870, 16384, 16384, 16374, 16368, 16382, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114678 dup_rows=131098 dup_rows_remote=120714 dedup_ratio=0.500099 router_tokens_per_expert=[7, 0, 0, 0, 4215, 20, 7329, 8171, 3486, 0, 8173, 9, 0, 93, 0, 3274, 352, 0, 0, 4855, 8169, 0, 0, 7969, 0, 0, 0, 0, 1, 0, 0, 7983, 0, 8192, 22, 14, 32, 0, 0, 3586, 0, 4400, 305, 812, 8157, 0, 0, 7923, 8188, 168, 3044, 5474, 1, 8124, 0, 0, 8192, 0, 1, 0, 221, 110, 0, 0] payload_rows_per_dst=[16384, 16382, 16368, 16384, 16384, 16378, 16382, 16384] dup_rows_per_dst=[23100, 21744, 10384, 15968, 23246, 10502, 9176, 16978] combine_rows_per_src=[16384, 16382, 16368, 16384, 16384, 16378, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130988 payload_rows_remote=114604 dup_rows=131156 dup_rows_remote=112380 dedup_ratio=0.500320 router_tokens_per_expert=[0, 0, 0, 32, 5589, 5, 27, 8192, 7751, 569, 4356, 0, 3, 4, 5, 2590, 1276, 0, 0, 6343, 8127, 0, 0, 8192, 0, 0, 0, 1163, 0, 0, 54, 8185, 0, 6832, 0, 38, 2, 0, 0, 574, 19, 8034, 138, 2, 8192, 0, 0, 8182, 8124, 0, 7807, 4098, 0, 8192, 32, 0, 8192, 0, 0, 0, 0, 151, 0, 0] payload_rows_per_dst=[16384, 16384, 16336, 16384, 16384, 16374, 16358, 16384] dup_rows_per_dst=[11306, 22288, 15156, 18776, 14892, 16376, 15676, 16686] combine_rows_per_src=[16384, 16384, 16336, 16384, 16384, 16374, 16358, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938819584 dispatch_meta_bytes=917080 combine_payload_bytes=938819584 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=939736664 ingress_bytes=1467301888 link_bytes=1467301888 token_bytes=1878556248 weight_movement_bytes=528482304 total_nvlink_bytes=2407038552 weight_to_token_ratio=0.281324 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=29440 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.216278091e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937525248 dispatch_meta_bytes=922768 combine_payload_bytes=937525248 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=938448016 ingress_bytes=1377927168 link_bytes=1377927168 token_bytes=1875973264 weight_movement_bytes=440401920 total_nvlink_bytes=2316375184 weight_to_token_ratio=0.234759 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.715816966e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=910508032 dispatch_meta_bytes=917424 combine_payload_bytes=910508032 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=911425456 ingress_bytes=1086668800 link_bytes=1086668800 token_bytes=1821933488 weight_movement_bytes=176160768 total_nvlink_bytes=1998094256 weight_to_token_ratio=0.096689 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.023674 gemm_active_groups=19 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.084891682e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129944 payload_rows_remote=113562 dup_rows=132200 dup_rows_remote=104644 dedup_ratio=0.504303 router_tokens_per_expert=[0, 9, 0, 3, 4257, 61, 7416, 7527, 287, 26, 7292, 497, 0, 219, 1, 0, 5877, 0, 2, 2729, 7639, 206, 0, 7968, 10, 642, 1, 96, 21, 7, 7, 8171, 0, 8172, 4, 18, 502, 244, 11, 773, 0, 6469, 0, 37, 8180, 124, 1475, 4057, 8076, 90, 3737, 6755, 0, 6929, 3, 0, 7147, 676, 1685, 0, 1415, 3417, 1, 104] payload_rows_per_dst=[16382, 15578, 16384, 16384, 16384, 16382, 16066, 16384] dup_rows_per_dst=[27556, 6768, 40412, 17872, 3064, 17332, 6690, 12506] combine_rows_per_src=[16382, 15578, 16384, 16384, 16384, 16382, 16066, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130982 payload_rows_remote=114672 dup_rows=131162 dup_rows_remote=110108 dedup_ratio=0.500343 router_tokens_per_expert=[0, 0, 6, 3, 8171, 47, 7980, 8120, 2515, 409, 7821, 0, 2, 0, 0, 3882, 91, 20, 151, 4096, 7553, 498, 0, 8132, 0, 1, 1, 6, 34, 19, 63, 8155, 0, 8058, 15, 144, 0, 0, 0, 0, 2, 6988, 0, 1617, 8180, 0, 0, 5937, 8071, 55, 4119, 3554, 1, 8055, 5, 0, 8184, 41, 0, 2, 0, 22, 0, 246] payload_rows_per_dst=[16384, 16310, 16384, 16384, 16382, 16380, 16374, 16384] dup_rows_per_dst=[32270, 21054, 11958, 16156, 12792, 18468, 7258, 11206] combine_rows_per_src=[16384, 16310, 16384, 16384, 16382, 16380, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131002 payload_rows_remote=114618 dup_rows=131142 dup_rows_remote=106678 dedup_ratio=0.500267 router_tokens_per_expert=[3, 17, 73, 0, 3890, 0, 3350, 7759, 4094, 0, 7822, 0, 0, 4, 144, 283, 4091, 39, 0, 131, 8010, 332, 43, 7999, 2, 31, 27, 0, 3, 7, 0, 8149, 67, 8189, 161, 0, 1, 4, 0, 4003, 34, 5896, 0, 457, 8179, 0, 0, 8061, 8077, 665, 3555, 7678, 5, 8138, 1, 3, 8120, 28, 0, 3, 7, 3390, 24, 23] payload_rows_per_dst=[16376, 16360, 16384, 16366, 16384, 16364, 16384, 16384] dup_rows_per_dst=[13808, 8334, 8908, 16176, 24464, 12532, 23756, 23164] combine_rows_per_src=[16376, 16360, 16384, 16366, 16384, 16364, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=941568 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940383744 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879825920 weight_movement_bytes=352321536 total_nvlink_bytes=2232147456 weight_to_token_ratio=0.187422 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.022727 gemm_active_groups=18 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938835968 dispatch_meta_bytes=907936 combine_payload_bytes=938835968 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939743904 ingress_bytes=1114996736 link_bytes=1114996736 token_bytes=1878579872 weight_movement_bytes=176160768 total_nvlink_bytes=2054740640 weight_to_token_ratio=0.093773 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243516299e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=930299904 dispatch_meta_bytes=872824 combine_payload_bytes=930299904 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=931172728 ingress_bytes=1370701824 link_bytes=1370701824 token_bytes=1861472632 weight_movement_bytes=440401920 total_nvlink_bytes=2301874552 weight_to_token_ratio=0.236588 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=25728 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.675358055e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=899120 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940292144 ingress_bytes=1115553792 link_bytes=1115553792 token_bytes=1879685168 weight_movement_bytes=176160768 total_nvlink_bytes=2055845936 weight_to_token_ratio=0.093718 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=28416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246635580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938950656 dispatch_meta_bytes=885184 combine_payload_bytes=938950656 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939835840 ingress_bytes=1115111424 link_bytes=1115111424 token_bytes=1878786496 weight_movement_bytes=176160768 total_nvlink_bytes=2054947264 weight_to_token_ratio=0.093763 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244158504e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130324 payload_rows_remote=113940 dup_rows=131820 dup_rows_remote=107922 dedup_ratio=0.502853 router_tokens_per_expert=[0, 211, 214, 1064, 0, 3682, 586, 0, 0, 0, 127, 7487, 3, 0, 50, 8153, 68, 7449, 0, 8141, 7966, 2, 8192, 0, 8, 0, 9, 4076, 4361, 0, 0, 7760, 0, 1020, 0, 25, 30, 7287, 159, 1745, 1, 7448, 25, 0, 113, 0, 6266, 0, 13, 4093, 0, 704, 7137, 0, 2, 8192, 0, 7993, 8192, 0, 0, 1018, 0, 0] payload_rows_per_dst=[16380, 16384, 16068, 15976, 16364, 16384, 16384, 16384] dup_rows_per_dst=[11414, 15256, 6712, 9048, 11572, 35898, 23898, 18022] combine_rows_per_src=[16380, 16384, 16068, 15976, 16364, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130028 payload_rows_remote=113644 dup_rows=132116 dup_rows_remote=107514 dedup_ratio=0.503983 router_tokens_per_expert=[2, 0, 0, 17, 0, 7503, 1030, 8, 0, 0, 461, 5847, 15, 393, 0, 8157, 12, 7712, 1, 8186, 7838, 1171, 8160, 0, 1887, 0, 9, 2491, 7798, 0, 0, 6030, 739, 373, 0, 219, 0, 8192, 131, 2007, 0, 7065, 0, 0, 0, 60, 496, 3, 193, 3717, 85, 0, 4544, 1, 0, 8182, 78, 8176, 8118, 3806, 0, 151, 0, 8] payload_rows_per_dst=[16384, 16352, 15722, 16036, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16766, 13394, 9898, 12678, 14342, 23374, 17062, 24602] combine_rows_per_src=[16384, 16352, 15722, 16036, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130760 payload_rows_remote=114384 dup_rows=131384 dup_rows_remote=125324 dedup_ratio=0.501190 router_tokens_per_expert=[26, 70, 0, 143, 0, 4121, 163, 0, 0, 0, 609, 1594, 0, 44, 0, 8184, 0, 8155, 2, 8192, 7934, 0, 8124, 0, 1163, 3, 0, 3776, 5222, 403, 0, 8184, 0, 3850, 0, 0, 128, 4433, 0, 6871, 0, 23, 0, 0, 0, 0, 2792, 0, 0, 4022, 16, 73, 7055, 0, 89, 7716, 2, 7954, 8084, 3733, 0, 8098, 20, 1] payload_rows_per_dst=[16384, 16384, 16310, 16384, 16300, 16376, 16238, 16384] dup_rows_per_dst=[9046, 4478, 15314, 19488, 15894, 6060, 21704, 39400] combine_rows_per_src=[16384, 16384, 16310, 16384, 16300, 16376, 16238, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130312 payload_rows_remote=113928 dup_rows=131832 dup_rows_remote=107740 dedup_ratio=0.502899 router_tokens_per_expert=[0, 35, 0, 175, 0, 4856, 28, 0, 0, 0, 828, 5182, 0, 0, 0, 8192, 0, 8191, 0, 8182, 8189, 1650, 8116, 0, 8, 0, 8, 2553, 8082, 0, 5, 8030, 0, 340, 0, 0, 0, 7609, 22, 6912, 0, 4598, 0, 1, 0, 20, 4067, 3, 10, 4022, 25, 1705, 4236, 0, 3, 8191, 0, 8137, 8173, 334, 0, 349, 5, 0] payload_rows_per_dst=[16382, 16384, 16384, 16382, 15744, 16302, 16384, 16350] dup_rows_per_dst=[12900, 17108, 24092, 20990, 14848, 17308, 12854, 11732] combine_rows_per_src=[16382, 16384, 16384, 16382, 15744, 16302, 16384, 16350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130814 payload_rows_remote=114518 dup_rows=131330 dup_rows_remote=116846 dedup_ratio=0.500984 router_tokens_per_expert=[0, 0, 0, 0, 0, 4985, 1947, 50, 0, 0, 1133, 6454, 1, 0, 0, 8173, 0, 8192, 0, 8191, 7656, 0, 8192, 0, 35, 83, 0, 54, 7791, 0, 0, 7427, 0, 109, 0, 0, 0, 7756, 4045, 1020, 0, 5585, 0, 0, 1, 0, 4094, 0, 0, 4086, 475, 179, 7016, 0, 154, 4071, 0, 8177, 8192, 303, 8, 5386, 51, 0] payload_rows_per_dst=[16384, 16356, 16382, 16296, 16244, 16384, 16384, 16384] dup_rows_per_dst=[13964, 15166, 15312, 14484, 9616, 19360, 15578, 27850] combine_rows_per_src=[16384, 16356, 16382, 16296, 16244, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=933396480 dispatch_meta_bytes=887448 combine_payload_bytes=933396480 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=934283928 ingress_bytes=1285718016 link_bytes=1285718016 token_bytes=1867680408 weight_movement_bytes=352321536 total_nvlink_bytes=2220001944 weight_to_token_ratio=0.188641 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=27008 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.199484204e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[4, 0, 0, 0, 0, 3, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=930971648 dispatch_meta_bytes=884632 combine_payload_bytes=930971648 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=931856280 ingress_bytes=1547534336 link_bytes=1547534336 token_bytes=1862827928 weight_movement_bytes=616562688 total_nvlink_bytes=2479390616 weight_to_token_ratio=0.330982 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=27776 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.665546309e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937033728 dispatch_meta_bytes=958832 combine_payload_bytes=937033728 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=937992560 ingress_bytes=1201274880 link_bytes=1201274880 token_bytes=1875026288 weight_movement_bytes=264241152 total_nvlink_bytes=2139267440 weight_to_token_ratio=0.140927 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.726637891e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=110164 payload_rows_remote=93780 dup_rows=151980 dup_rows_remote=125514 dedup_ratio=0.579758 router_tokens_per_expert=[2, 19, 6, 8, 22, 7962, 1, 93, 0, 3, 1517, 8188, 694, 3, 0, 8190, 0, 7868, 0, 8180, 7700, 3821, 8191, 8, 8, 165, 29, 4103, 5213, 1056, 0, 4337, 25, 4399, 0, 0, 5, 3457, 294, 3785, 4, 4089, 0, 0, 2, 0, 502, 2, 55, 4083, 11, 2445, 8094, 1, 0, 5967, 10, 7306, 7628, 1, 0, 1226, 0, 294] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 9146, 16204, 2898] dup_rows_per_dst=[26466, 33484, 27976, 21614, 15738, 9068, 17470, 164] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 9146, 16204, 2898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125724 payload_rows_remote=109340 dup_rows=136420 dup_rows_remote=115478 dedup_ratio=0.520401 router_tokens_per_expert=[9, 0, 98, 7, 0, 5321, 3294, 0, 184, 1, 306, 5685, 380, 2, 0, 8017, 25, 8169, 60, 7654, 8165, 0, 8130, 25, 5508, 0, 70, 0, 7923, 0, 0, 126, 1, 92, 0, 1, 0, 8134, 4104, 7042, 6, 4154, 0, 43, 25, 0, 4159, 0, 0, 166, 0, 4085, 4257, 0, 0, 4638, 6, 5413, 8128, 0, 123, 7328, 5, 3] payload_rows_per_dst=[16384, 16384, 16330, 16382, 16384, 16384, 11746, 15730] dup_rows_per_dst=[25604, 20942, 15526, 17282, 30438, 16650, 3416, 6562] combine_rows_per_src=[16384, 16384, 16330, 16382, 16384, 16384, 11746, 15730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114640 dup_rows=131122 dup_rows_remote=111608 dedup_ratio=0.500191 router_tokens_per_expert=[5, 0, 0, 78, 1, 7205, 197, 1, 107, 11, 1461, 4018, 51, 0, 9, 8187, 0, 8187, 0, 8092, 7983, 746, 8160, 23, 29, 0, 0, 4664, 7877, 4, 27, 3759, 20, 4175, 0, 0, 2, 8150, 5, 5596, 0, 4654, 3, 0, 68, 0, 648, 0, 36, 4130, 236, 3559, 4124, 0, 267, 7868, 25, 8192, 8171, 0, 4, 172, 83, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16372, 16348, 16384] dup_rows_per_dst=[14918, 11304, 17350, 16336, 19514, 10694, 24092, 16914] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16372, 16348, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=933298176 dispatch_meta_bytes=886672 combine_payload_bytes=933298176 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=934184848 ingress_bytes=1726021632 link_bytes=1726021632 token_bytes=1867483024 weight_movement_bytes=792723456 total_nvlink_bytes=2660206480 weight_to_token_ratio=0.424488 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.023674 gemm_active_groups=16 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.665000662e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938131456 dispatch_meta_bytes=925456 combine_payload_bytes=938131456 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=939056912 ingress_bytes=1202372608 link_bytes=1202372608 token_bytes=1877188368 weight_movement_bytes=264241152 total_nvlink_bytes=2141429520 weight_to_token_ratio=0.140764 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=25984 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.732784710e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29467044216 prefetch_bytes=3170893824 reduce_bytes=3170893824 weight_movement_bytes=6341787648 egress_bytes=14740777336 ingress_bytes=21068054528 link_bytes=21068054528 total_nvlink_bytes=35808831864 weight_to_token_ratio=0.215216 prefetch_share_of_ingress=0.150507 dedup_ratio=0.509296 prefetch_slots=72 unused_prefetch_slots=440 slot_utilization=0.140625 reduce_slots=36 remote_demand=36 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021188 gemm_inner_pad_upper_bound=38989 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.373269445e-05 prefetch_bytes_per_gemm_flop=1.109730114e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29166593648 prefetch_bytes=2466250752 reduce_bytes=3699376128 weight_movement_bytes=6165626880 egress_bytes=14590387824 ingress_bytes=20741832704 link_bytes=20741832704 total_nvlink_bytes=35332220528 weight_to_token_ratio=0.211393 prefetch_share_of_ingress=0.118902 dedup_ratio=0.513868 prefetch_slots=56 unused_prefetch_slots=456 slot_utilization=0.109375 reduce_slots=42 remote_demand=28 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021011 gemm_inner_pad_upper_bound=39497 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.259100317e-05 prefetch_bytes_per_gemm_flop=8.631234217e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29499970128 prefetch_bytes=3170893824 reduce_bytes=1497366528 weight_movement_bytes=4668260352 egress_bytes=14757286480 ingress_bytes=19410944000 link_bytes=19410944000 total_nvlink_bytes=34168230480 weight_to_token_ratio=0.158246 prefetch_share_of_ingress=0.163356 dedup_ratio=0.509548 prefetch_slots=72 unused_prefetch_slots=440 slot_utilization=0.140625 reduce_slots=17 remote_demand=36 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=39497 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.793323992e-05 prefetch_bytes_per_gemm_flop=1.109730114e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=768245760 dispatch_meta_bytes=877176 combine_payload_bytes=768245760 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=769122936 ingress_bytes=1120567296 link_bytes=1120567296 token_bytes=1537368696 weight_movement_bytes=352321536 total_nvlink_bytes=1889690232 weight_to_token_ratio=0.229172 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.018939 gemm_active_groups=22 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.274709110e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29667255096 prefetch_bytes=2554331136 reduce_bytes=4580179968 weight_movement_bytes=7134511104 egress_bytes=14840947512 ingress_bytes=21960818688 link_bytes=21960818688 total_nvlink_bytes=36801766200 weight_to_token_ratio=0.240484 prefetch_share_of_ingress=0.116313 dedup_ratio=0.506373 prefetch_slots=58 unused_prefetch_slots=454 slot_utilization=0.113281 reduce_slots=52 remote_demand=29 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021603 gemm_inner_pad_upper_bound=37973 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.685713610e-05 prefetch_bytes_per_gemm_flop=8.939492582e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29836318360 prefetch_bytes=3082813440 reduce_bytes=1937768448 weight_movement_bytes=5020581888 egress_bytes=14925567640 ingress_bytes=19931332608 link_bytes=19931332608 total_nvlink_bytes=34856900248 weight_to_token_ratio=0.168271 prefetch_share_of_ingress=0.154672 dedup_ratio=0.504767 prefetch_slots=70 unused_prefetch_slots=442 slot_utilization=0.136719 reduce_slots=22 remote_demand=35 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=39624 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.975446429e-05 prefetch_bytes_per_gemm_flop=1.078904277e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=895713280 dispatch_meta_bytes=899272 combine_payload_bytes=895713280 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=896612552 ingress_bytes=1071874048 link_bytes=1071874048 token_bytes=1792325832 weight_movement_bytes=176160768 total_nvlink_bytes=1968486600 weight_to_token_ratio=0.098286 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.002047247e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=904992 combine_payload_bytes=939130880 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940035872 ingress_bytes=1115291648 link_bytes=1115291648 token_bytes=1879166752 weight_movement_bytes=176160768 total_nvlink_bytes=2055327520 weight_to_token_ratio=0.093744 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.022727 gemm_active_groups=17 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245167683e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=28747058472 prefetch_bytes=2818572288 reduce_bytes=2730491904 weight_movement_bytes=5549064192 egress_bytes=14380600616 ingress_bytes=19915522048 link_bytes=19915522048 total_nvlink_bytes=34296122664 weight_to_token_ratio=0.193031 prefetch_share_of_ingress=0.141526 dedup_ratio=0.519851 prefetch_slots=64 unused_prefetch_slots=448 slot_utilization=0.125000 reduce_slots=31 remote_demand=32 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021011 gemm_inner_pad_upper_bound=39243 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.969913145e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29706045704 prefetch_bytes=2378170368 reduce_bytes=4492099584 weight_movement_bytes=6870269952 egress_bytes=14860421384 ingress_bytes=21715894272 link_bytes=21715894272 total_nvlink_bytes=36576315656 weight_to_token_ratio=0.231275 prefetch_share_of_ingress=0.109513 dedup_ratio=0.507061 prefetch_slots=54 unused_prefetch_slots=458 slot_utilization=0.105469 reduce_slots=51 remote_demand=27 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021544 gemm_inner_pad_upper_bound=37973 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.599996454e-05 prefetch_bytes_per_gemm_flop=8.322975852e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29244820720 prefetch_bytes=3963617280 reduce_bytes=1497366528 weight_movement_bytes=5460983808 egress_bytes=14629637360 ingress_bytes=20076167168 link_bytes=20076167168 total_nvlink_bytes=34705804528 weight_to_token_ratio=0.186733 prefetch_share_of_ingress=0.197429 dedup_ratio=0.512504 prefetch_slots=90 unused_prefetch_slots=422 slot_utilization=0.175781 reduce_slots=17 remote_demand=45 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020419 gemm_inner_pad_upper_bound=41021 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.026134746e-05 prefetch_bytes_per_gemm_flop=1.387162642e-05 anomalies=none - [2026-09-16 21:39:29] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 616.9 | throughput per GPU (TFLOP/s/GPU): 540.4 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.051970E+01 | loss scale: 1.0 | grad norm: 215.088 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122842 payload_rows_remote=109016 dup_rows=139302 dup_rows_remote=128622 dedup_ratio=0.531395 router_tokens_per_expert=[111, 1867, 3968, 72, 3321, 11, 1993, 64, 4297, 165, 3238, 1490, 1159, 873, 92, 891, 480, 3117, 4139, 2538, 7015, 2809, 1277, 3494, 16, 6283, 355, 1707, 854, 215, 1624, 3031, 6801, 408, 212, 1893, 1724, 253, 1462, 2721, 4719, 6007, 2070, 686, 441, 4708, 2576, 1215, 525, 1166, 1964, 338, 472, 1094, 6553, 514, 59, 3778, 7145, 3164, 1129, 1281, 1193, 235] payload_rows_per_dst=[15808, 16126, 14364, 15536, 15144, 13826, 15892, 16146] dup_rows_per_dst=[18260, 22314, 12824, 20912, 15804, 10680, 18686, 19822] combine_rows_per_src=[15808, 16126, 14364, 15536, 15144, 13826, 15892, 16146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120824 payload_rows_remote=104976 dup_rows=141320 dup_rows_remote=121832 dedup_ratio=0.539093 router_tokens_per_expert=[85, 1708, 4360, 93, 3166, 14, 1728, 52, 2355, 200, 2509, 1535, 1127, 732, 46, 968, 458, 3814, 4242, 3339, 6979, 2111, 1501, 2969, 6, 6624, 334, 2102, 1040, 403, 1277, 3026, 7376, 296, 400, 1365, 1042, 316, 2381, 2574, 4714, 6159, 1957, 837, 399, 5016, 3030, 3731, 398, 1082, 1971, 316, 241, 579, 6601, 461, 65, 3281, 5985, 3641, 398, 1707, 1689, 161] payload_rows_per_dst=[12830, 14220, 14838, 15896, 15052, 16122, 16018, 15848] dup_rows_per_dst=[9582, 12702, 15440, 22212, 14966, 29334, 17596, 19488] combine_rows_per_src=[12830, 14220, 14838, 15896, 15052, 16122, 16018, 15848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125104 payload_rows_remote=109400 dup_rows=137040 dup_rows_remote=116266 dedup_ratio=0.522766 router_tokens_per_expert=[102, 2388, 4576, 68, 3242, 4, 1502, 94, 3399, 110, 2278, 976, 1214, 874, 25, 1044, 455, 3950, 4480, 3101, 6842, 1577, 1625, 3051, 4, 7018, 329, 2104, 947, 295, 1265, 3113, 7168, 450, 222, 1082, 1757, 169, 2360, 2301, 4849, 6333, 1449, 597, 374, 3612, 3116, 3149, 455, 1436, 1908, 280, 160, 908, 6195, 583, 89, 3167, 6921, 4300, 821, 1586, 1019, 204] payload_rows_per_dst=[16048, 16038, 15704, 15692, 15398, 14492, 15588, 16144] dup_rows_per_dst=[20570, 17486, 20774, 14458, 15620, 10970, 17092, 20070] combine_rows_per_src=[16048, 16038, 15704, 15692, 15398, 14492, 15588, 16144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123944 payload_rows_remote=108708 dup_rows=138200 dup_rows_remote=125578 dedup_ratio=0.527191 router_tokens_per_expert=[66, 1894, 4285, 224, 3656, 14, 1539, 75, 3499, 123, 2222, 1455, 1791, 518, 51, 1008, 544, 3892, 4702, 2675, 6818, 2380, 1706, 3284, 17, 6486, 323, 1835, 882, 155, 1441, 2495, 7200, 351, 302, 1871, 1442, 182, 2371, 2511, 4624, 6036, 1370, 549, 413, 4829, 2227, 2055, 466, 1290, 2444, 238, 341, 1087, 7193, 444, 92, 3674, 6977, 3455, 623, 1323, 809, 228] payload_rows_per_dst=[15940, 15926, 15694, 15236, 15408, 13542, 16116, 16082] dup_rows_per_dst=[19638, 19044, 22082, 12622, 17052, 8520, 20962, 18280] combine_rows_per_src=[15940, 15926, 15694, 15236, 15408, 13542, 16116, 16082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124720 payload_rows_remote=108858 dup_rows=137424 dup_rows_remote=122798 dedup_ratio=0.524231 router_tokens_per_expert=[46, 2293, 4827, 170, 4089, 7, 1516, 76, 2432, 172, 1948, 1157, 1129, 565, 67, 784, 481, 3944, 4113, 2694, 6990, 2046, 1343, 2724, 47, 6124, 441, 1951, 990, 470, 1093, 3332, 7327, 375, 335, 1884, 1176, 199, 2436, 2670, 5036, 5603, 2279, 542, 382, 4629, 3634, 3660, 274, 1307, 2073, 207, 204, 1078, 6372, 731, 33, 2844, 5751, 4621, 518, 1673, 991, 167] payload_rows_per_dst=[16022, 15862, 15494, 15252, 15498, 16012, 15702, 14878] dup_rows_per_dst=[21232, 14626, 19196, 13644, 17306, 23608, 15192, 12620] combine_rows_per_src=[16022, 15862, 15494, 15252, 15498, 16012, 15702, 14878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122070 payload_rows_remote=106224 dup_rows=140074 dup_rows_remote=120374 dedup_ratio=0.534340 router_tokens_per_expert=[73, 1543, 4083, 133, 3619, 5, 1907, 64, 3402, 173, 2433, 1302, 1014, 728, 40, 907, 401, 4451, 4058, 2892, 7149, 2131, 1516, 2907, 10, 6682, 306, 1604, 644, 132, 1317, 3409, 7234, 518, 220, 1827, 928, 243, 2240, 2652, 4838, 6370, 1504, 380, 442, 4784, 3297, 2230, 380, 1459, 2200, 280, 307, 1155, 6564, 322, 103, 3311, 7074, 4019, 378, 1739, 720, 319] payload_rows_per_dst=[13366, 16092, 13718, 15810, 15416, 15656, 15846, 16166] dup_rows_per_dst=[11444, 18204, 10582, 20514, 16308, 24162, 19700, 19160] combine_rows_per_src=[13366, 16092, 13718, 15810, 15416, 15656, 15846, 16166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124936 payload_rows_remote=109208 dup_rows=137208 dup_rows_remote=118112 dedup_ratio=0.523407 router_tokens_per_expert=[53, 1628, 4700, 160, 4195, 5, 1809, 39, 2868, 126, 1694, 667, 1380, 478, 49, 1101, 426, 4816, 4137, 3372, 6889, 1642, 1688, 3192, 7, 5958, 338, 1602, 831, 95, 1151, 3257, 7472, 890, 246, 1798, 1322, 243, 2676, 2765, 5247, 5461, 1370, 256, 373, 4492, 3196, 4062, 337, 1029, 2347, 223, 163, 623, 6705, 538, 163, 2861, 6739, 4250, 494, 1273, 947, 158] payload_rows_per_dst=[15960, 15864, 15130, 15266, 15728, 15398, 15828, 15762] dup_rows_per_dst=[20140, 14640, 16162, 18466, 19096, 12946, 17750, 18008] combine_rows_per_src=[15960, 15864, 15130, 15266, 15728, 15398, 15828, 15762] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=893059072 dispatch_meta_bytes=950552 combine_payload_bytes=893059072 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=894009624 ingress_bytes=1421541376 link_bytes=1421541376 token_bytes=1787068696 weight_movement_bytes=528482304 total_nvlink_bytes=2315551000 weight_to_token_ratio=0.295726 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.023674 gemm_active_groups=20 gemm_peak_group_rows=18688 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.960038325e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=896204800 dispatch_meta_bytes=902664 combine_payload_bytes=896204800 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=897107464 ingress_bytes=1512767488 link_bytes=1512767488 token_bytes=1793312264 weight_movement_bytes=616562688 total_nvlink_bytes=2409874952 weight_to_token_ratio=0.343812 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=15232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.470866473e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=859963392 dispatch_meta_bytes=907232 combine_payload_bytes=859963392 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=860870624 ingress_bytes=1300365312 link_bytes=1300365312 token_bytes=1720834016 weight_movement_bytes=440401920 total_nvlink_bytes=2161235936 weight_to_token_ratio=0.255924 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=23936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.281502948e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=890535936 dispatch_meta_bytes=937144 combine_payload_bytes=890535936 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=891473080 ingress_bytes=1066696704 link_bytes=1066696704 token_bytes=1782009016 weight_movement_bytes=176160768 total_nvlink_bytes=1958169784 weight_to_token_ratio=0.098855 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=25856 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.973056281e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=891764736 dispatch_meta_bytes=926624 combine_payload_bytes=891764736 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=892691360 ingress_bytes=1067925504 link_bytes=1067925504 token_bytes=1784456096 weight_movement_bytes=176160768 total_nvlink_bytes=1960616864 weight_to_token_ratio=0.098720 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=26368 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.979937048e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122914 payload_rows_remote=106942 dup_rows=139230 dup_rows_remote=119898 dedup_ratio=0.531120 router_tokens_per_expert=[72, 1988, 4907, 180, 2700, 30, 1609, 71, 2881, 266, 2683, 1177, 1104, 626, 97, 841, 463, 3949, 4348, 2757, 6825, 1744, 1278, 2889, 12, 5767, 330, 2131, 1249, 366, 1549, 3910, 7079, 518, 348, 2353, 1689, 231, 1960, 3384, 5058, 6095, 1746, 649, 652, 4754, 2780, 2705, 443, 1446, 1374, 245, 181, 860, 6329, 315, 76, 3217, 6615, 3266, 664, 1427, 1598, 216] payload_rows_per_dst=[15972, 15906, 15726, 15450, 15436, 15742, 14668, 14014] dup_rows_per_dst=[19332, 17094, 24416, 15178, 22912, 21134, 9508, 9656] combine_rows_per_src=[15972, 15906, 15726, 15450, 15436, 15742, 14668, 14014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=870187008 dispatch_meta_bytes=906392 combine_payload_bytes=870187008 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=871093400 ingress_bytes=1046347776 link_bytes=1046347776 token_bytes=1741280408 weight_movement_bytes=176160768 total_nvlink_bytes=1917441176 weight_to_token_ratio=0.101167 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=26368 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.859110779e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=894631936 dispatch_meta_bytes=909280 combine_payload_bytes=894631936 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=895541216 ingress_bytes=1070792704 link_bytes=1070792704 token_bytes=1790173152 weight_movement_bytes=176160768 total_nvlink_bytes=1966333920 weight_to_token_ratio=0.098404 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=28928 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.995992172e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=876068864 dispatch_meta_bytes=907360 combine_payload_bytes=876068864 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=876976224 ingress_bytes=1052229632 link_bytes=1052229632 token_bytes=1753045088 weight_movement_bytes=176160768 total_nvlink_bytes=1929205856 weight_to_token_ratio=0.100488 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=18560 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.892046717e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129308 payload_rows_remote=112926 dup_rows=132836 dup_rows_remote=110348 dedup_ratio=0.506729 router_tokens_per_expert=[1, 0, 1, 3, 5, 2, 1, 8182, 7209, 5690, 7759, 0, 8039, 273, 7670, 4966, 4, 3, 3, 5682, 5223, 8188, 41, 8062, 729, 7165, 7, 5, 0, 0, 7994, 0, 37, 19, 0, 8086, 2, 0, 0, 0, 677, 372, 12, 4, 196, 0, 3698, 25, 0, 0, 51, 8155, 0, 7775, 0, 0, 12, 6, 5, 805, 4068, 4156, 0, 4] payload_rows_per_dst=[16384, 14866, 16318, 16370, 16244, 16382, 16360, 16384] dup_rows_per_dst=[15532, 8608, 20622, 16526, 15376, 22488, 15602, 18082] combine_rows_per_src=[16384, 14866, 16318, 16370, 16244, 16382, 16360, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130564 payload_rows_remote=114566 dup_rows=131580 dup_rows_remote=113716 dedup_ratio=0.501938 router_tokens_per_expert=[20, 0, 0, 0, 7, 49, 0, 8167, 7140, 4597, 7588, 28, 8095, 3454, 8176, 4968, 5, 1, 7, 7882, 4164, 8184, 0, 7713, 5869, 4838, 1, 0, 12, 0, 7668, 29, 21, 198, 14, 7353, 0, 0, 1, 0, 371, 19, 71, 26, 384, 0, 2310, 1, 0, 67, 26, 8050, 2, 8105, 18, 0, 67, 16, 0, 356, 691, 4240, 3, 0] payload_rows_per_dst=[16382, 16382, 15998, 16324, 16376, 16370, 16358, 16374] dup_rows_per_dst=[15692, 29674, 17864, 20510, 15168, 6196, 16178, 10298] combine_rows_per_src=[16382, 16382, 15998, 16324, 16376, 16370, 16358, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130066 payload_rows_remote=113738 dup_rows=132078 dup_rows_remote=119716 dedup_ratio=0.503838 router_tokens_per_expert=[5, 0, 3, 1, 0, 9, 2, 8186, 6096, 5761, 8057, 0, 8178, 749, 7874, 4732, 1, 0, 0, 7528, 6201, 8183, 0, 7438, 352, 5865, 0, 0, 1, 0, 8127, 0, 22, 0, 6, 8184, 8, 4, 0, 0, 18, 121, 2, 154, 1795, 0, 6800, 22, 0, 3, 4, 7898, 1, 7654, 4, 0, 1, 24, 10, 364, 3588, 1034, 0, 2] payload_rows_per_dst=[16382, 15756, 16374, 16328, 16384, 16384, 16080, 16378] dup_rows_per_dst=[16266, 13806, 25962, 12362, 16412, 22296, 15048, 9926] combine_rows_per_src=[16382, 15756, 16374, 16328, 16384, 16384, 16080, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128006 payload_rows_remote=111720 dup_rows=134138 dup_rows_remote=115662 dedup_ratio=0.511696 router_tokens_per_expert=[6, 0, 0, 3, 0, 14, 7, 7922, 3752, 6333, 4924, 1, 8152, 514, 8182, 6833, 0, 3, 94, 8039, 5362, 8099, 55, 6065, 598, 6825, 2, 5, 5, 0, 6738, 0, 5, 7, 0, 8182, 3, 0, 1, 56, 0, 0, 3, 169, 2062, 6, 6068, 52, 0, 9, 6, 8153, 2, 7943, 62, 18, 36, 4, 0, 1052, 5665, 2965, 8, 2] payload_rows_per_dst=[16096, 14244, 15932, 16296, 16384, 16384, 16384, 16286] dup_rows_per_dst=[12972, 3854, 9862, 18114, 19238, 29554, 22068, 18476] combine_rows_per_src=[16096, 14244, 15932, 16296, 16384, 16384, 16384, 16286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130554 payload_rows_remote=114170 dup_rows=131590 dup_rows_remote=100036 dedup_ratio=0.501976 router_tokens_per_expert=[0, 1, 13, 3, 1, 74, 0, 8076, 4442, 6619, 8076, 2, 7998, 3021, 8178, 5439, 0, 0, 15, 7966, 3113, 8086, 26, 8077, 763, 5227, 14, 0, 98, 0, 7966, 0, 112, 0, 6, 8008, 6, 0, 2, 0, 137, 47, 0, 13, 492, 3, 3863, 105, 0, 138, 0, 8007, 0, 8065, 14, 1, 0, 5, 6, 181, 3940, 4625, 2, 0] payload_rows_per_dst=[16382, 16384, 16356, 16276, 16338, 16238, 16200, 16380] dup_rows_per_dst=[17360, 31554, 12058, 11860, 16084, 9160, 16250, 17264] combine_rows_per_src=[16382, 16384, 16356, 16276, 16338, 16238, 16200, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 2, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 6, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=11 export_slots=11 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 4, 0, 3] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=925089792 dispatch_meta_bytes=893096 combine_payload_bytes=925089792 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=925982888 ingress_bytes=1277411328 link_bytes=1277411328 token_bytes=1851072680 weight_movement_bytes=352321536 total_nvlink_bytes=2203394216 weight_to_token_ratio=0.190334 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.018939 gemm_active_groups=22 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.152970219e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130134 payload_rows_remote=113756 dup_rows=132010 dup_rows_remote=115834 dedup_ratio=0.503578 router_tokens_per_expert=[6, 0, 6, 9, 0, 1, 0, 8145, 6289, 5223, 7649, 0, 8171, 105, 8187, 5621, 0, 0, 3, 8112, 2455, 8191, 3, 8125, 191, 7410, 0, 0, 7, 0, 8162, 1, 4, 1, 5, 8187, 9, 0, 0, 13, 239, 9, 2, 5, 511, 0, 3316, 2, 0, 6, 0, 8177, 1, 8077, 16, 0, 1, 2, 1, 760, 5819, 3810, 1, 26] payload_rows_per_dst=[16352, 15596, 16296, 16368, 16384, 16382, 16378, 16378] dup_rows_per_dst=[15286, 6126, 8288, 27986, 16438, 20912, 16176, 20798] combine_rows_per_src=[16352, 15596, 16296, 16368, 16384, 16382, 16378, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938524672 dispatch_meta_bytes=913128 combine_payload_bytes=938524672 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=939437800 ingress_bytes=1819328512 link_bytes=1819328512 token_bytes=1877962472 weight_movement_bytes=880803840 total_nvlink_bytes=2758766312 weight_to_token_ratio=0.469021 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=24832 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018748024e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=931741696 dispatch_meta_bytes=933816 combine_payload_bytes=931741696 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=932675512 ingress_bytes=1107902464 link_bytes=1107902464 token_bytes=1864417208 weight_movement_bytes=176160768 total_nvlink_bytes=2040577976 weight_to_token_ratio=0.094486 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.203791337e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=915210240 dispatch_meta_bytes=909528 combine_payload_bytes=915210240 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=916119768 ingress_bytes=1179451392 link_bytes=1179451392 token_bytes=1831330008 weight_movement_bytes=264241152 total_nvlink_bytes=2095571160 weight_to_token_ratio=0.144289 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.018939 gemm_active_groups=21 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.604435468e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=935280640 dispatch_meta_bytes=856824 combine_payload_bytes=935280640 prefetch_bytes=176160768 reduce_bytes=968884224 egress_bytes=936137464 ingress_bytes=2080325632 link_bytes=2080325632 token_bytes=1871418104 weight_movement_bytes=1145044992 total_nvlink_bytes=3016463096 weight_to_token_ratio=0.611860 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=21760 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.164895517e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=931889152 dispatch_meta_bytes=918360 combine_payload_bytes=931889152 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=932807512 ingress_bytes=1108049920 link_bytes=1108049920 token_bytes=1864696664 weight_movement_bytes=176160768 total_nvlink_bytes=2040857432 weight_to_token_ratio=0.094472 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.204617029e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130628 payload_rows_remote=114244 dup_rows=131516 dup_rows_remote=107940 dedup_ratio=0.501694 router_tokens_per_expert=[59, 9, 7, 1, 6, 5, 17, 8132, 7550, 5235, 8160, 3, 7978, 1512, 8109, 4254, 3, 15, 49, 7594, 7207, 8141, 38, 7558, 445, 5858, 0, 12, 31, 14, 7861, 29, 271, 0, 8, 8186, 9, 4, 28, 1, 135, 24, 140, 44, 1066, 17, 1380, 168, 28, 34, 2, 7355, 2, 7774, 1, 6, 19, 0, 0, 158, 3125, 5186, 2, 7] payload_rows_per_dst=[16384, 16192, 16360, 16344, 16384, 16358, 16288, 16318] dup_rows_per_dst=[23576, 29384, 12950, 12156, 16934, 5666, 14116, 16734] combine_rows_per_src=[16384, 16192, 16360, 16344, 16384, 16358, 16288, 16318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130670 payload_rows_remote=114288 dup_rows=131474 dup_rows_remote=115856 dedup_ratio=0.501534 router_tokens_per_expert=[1, 1, 0, 0, 6, 1, 0, 8188, 3893, 6951, 7257, 5, 8045, 355, 8189, 6470, 0, 0, 3, 8139, 4304, 8184, 0, 8006, 350, 5635, 1, 4, 0, 0, 8172, 0, 13, 1, 0, 7796, 2, 0, 0, 0, 10, 4, 1, 14, 306, 0, 4342, 6, 0, 15, 2, 8128, 0, 8178, 6, 0, 0, 2, 0, 371, 4865, 4850, 0, 0] payload_rows_per_dst=[16384, 16000, 16384, 16374, 16382, 16384, 16382, 16380] dup_rows_per_dst=[14714, 13718, 24520, 11950, 15618, 14990, 16276, 19688] combine_rows_per_src=[16384, 16000, 16384, 16374, 16382, 16384, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130980 payload_rows_remote=114674 dup_rows=131164 dup_rows_remote=118254 dedup_ratio=0.500351 router_tokens_per_expert=[0, 1, 34, 1, 0, 0, 28, 0, 16, 8077, 2, 3, 1143, 8189, 8192, 84, 32, 8187, 0, 1, 8109, 3, 1, 0, 8184, 3, 11, 0, 0, 8060, 45, 3, 8186, 0, 33, 0, 7287, 581, 0, 2, 7689, 6809, 102, 2, 5, 0, 0, 1, 0, 5483, 8191, 0, 0, 4375, 0, 2, 8188, 14, 2764, 2, 4769, 2, 1, 8175] payload_rows_per_dst=[16384, 16384, 16380, 16374, 16384, 16306, 16384, 16384] dup_rows_per_dst=[16278, 18870, 16286, 16238, 15794, 12910, 19718, 15070] combine_rows_per_src=[16384, 16384, 16380, 16374, 16384, 16306, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131008 payload_rows_remote=114624 dup_rows=131136 dup_rows_remote=114878 dedup_ratio=0.500244 router_tokens_per_expert=[0, 0, 0, 8, 25, 1, 531, 14, 166, 8186, 0, 0, 961, 8025, 8163, 4, 131, 8180, 1, 0, 8002, 7, 0, 0, 8192, 2, 467, 0, 0, 8164, 1, 0, 8192, 0, 4, 0, 4514, 526, 0, 9, 6985, 6001, 12, 42, 0, 1, 0, 0, 237, 7718, 8192, 0, 0, 5530, 0, 9, 8191, 9, 2139, 0, 5623, 0, 0, 7907] payload_rows_per_dst=[16354, 16382, 16384, 16384, 16384, 16356, 16384, 16380] dup_rows_per_dst=[17448, 18366, 16258, 17268, 10106, 13384, 23330, 14976] combine_rows_per_src=[16354, 16382, 16384, 16384, 16384, 16356, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=935886848 dispatch_meta_bytes=888736 combine_payload_bytes=935886848 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936775584 ingress_bytes=1288208384 link_bytes=1288208384 token_bytes=1872662432 weight_movement_bytes=352321536 total_nvlink_bytes=2224983968 weight_to_token_ratio=0.188139 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.018939 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.213429226e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114666 dup_rows=131098 dup_rows_remote=111498 dedup_ratio=0.500099 router_tokens_per_expert=[0, 39, 2, 9, 6, 0, 2, 4, 39, 8184, 0, 72, 614, 8163, 8170, 7, 167, 8105, 0, 0, 8184, 0, 0, 2, 8188, 0, 1626, 5, 1, 8168, 2, 0, 8170, 0, 6, 0, 7875, 238, 10, 12, 7156, 7691, 8, 1, 2, 2, 1, 1, 1015, 2144, 8192, 0, 0, 4769, 0, 0, 8190, 3, 412, 0, 7576, 3, 1, 7835] payload_rows_per_dst=[16384, 16384, 16374, 16380, 16378, 16378, 16384, 16384] dup_rows_per_dst=[16462, 17772, 16542, 19600, 16244, 13346, 15856, 15276] combine_rows_per_src=[16384, 16384, 16374, 16380, 16378, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128608 payload_rows_remote=112228 dup_rows=133536 dup_rows_remote=93364 dedup_ratio=0.509399 router_tokens_per_expert=[2, 11, 10, 0, 32, 3, 1, 33, 0, 8192, 7, 93, 32, 8163, 8185, 1, 11, 8187, 24, 2, 8144, 1, 0, 5, 8139, 0, 483, 0, 0, 8090, 63, 10, 8121, 37, 0, 0, 4275, 3987, 0, 7, 8178, 7337, 171, 17, 0, 0, 14, 15, 274, 5323, 8175, 1, 2, 3631, 4, 20, 8184, 8, 30, 16, 7094, 0, 41, 8186] payload_rows_per_dst=[16384, 16384, 16384, 14554, 15756, 16384, 16382, 16380] dup_rows_per_dst=[14158, 3812, 16304, 13172, 6540, 15080, 24298, 40172] combine_rows_per_src=[16384, 16384, 16384, 14554, 15756, 16384, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=936247296 dispatch_meta_bytes=920576 combine_payload_bytes=936247296 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937167872 ingress_bytes=1288568832 link_bytes=1288568832 token_bytes=1873415168 weight_movement_bytes=352321536 total_nvlink_bytes=2225736704 weight_to_token_ratio=0.188064 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017992 gemm_active_groups=21 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.215447584e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127840 payload_rows_remote=111456 dup_rows=134304 dup_rows_remote=110038 dedup_ratio=0.512329 router_tokens_per_expert=[2, 7, 0, 4, 77, 19, 6, 11, 1, 7864, 2, 2, 4333, 7885, 8188, 0, 0, 8183, 0, 0, 8160, 9, 0, 1, 8181, 0, 456, 29, 0, 8004, 6, 1, 8125, 3, 4, 61, 3142, 5286, 11, 1, 7712, 7789, 32, 2, 0, 109, 1, 12, 121, 5727, 8189, 0, 0, 2620, 0, 0, 8151, 0, 1559, 0, 2892, 0, 41, 8051] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16378, 13310, 16232] dup_rows_per_dst=[16070, 24266, 16322, 16970, 20106, 23766, 7950, 8854] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16378, 13310, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=931712 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940341120 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879750528 weight_movement_bytes=176160768 total_nvlink_bytes=2055911296 weight_to_token_ratio=0.093715 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=30976 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=109560 dedup_ratio=0.500099 router_tokens_per_expert=[1, 17, 0, 13, 1, 0, 18, 1, 1, 8122, 0, 5, 777, 8001, 8188, 3, 49, 8192, 0, 0, 8190, 0, 1, 0, 8182, 5, 183, 0, 0, 8191, 2, 0, 8158, 0, 0, 12, 6879, 3925, 4, 0, 7494, 7554, 4, 0, 0, 22, 1, 14, 263, 6220, 8192, 0, 0, 4286, 0, 0, 8192, 0, 193, 0, 3320, 7, 0, 8189] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16370, 16372, 16384, 16384] dup_rows_per_dst=[16464, 14010, 16480, 16742, 16948, 13806, 21538, 15110] combine_rows_per_src=[16384, 16384, 16384, 16384, 16370, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[5, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=938999808 dispatch_meta_bytes=918008 combine_payload_bytes=938999808 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=939917816 ingress_bytes=1115160576 link_bytes=1115160576 token_bytes=1878917624 weight_movement_bytes=176160768 total_nvlink_bytes=2055078392 weight_to_token_ratio=0.093757 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244433735e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=904656 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940248528 ingress_bytes=1379745792 link_bytes=1379745792 token_bytes=1879592400 weight_movement_bytes=440401920 total_nvlink_bytes=2319994320 weight_to_token_ratio=0.234307 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726000501e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=919371776 dispatch_meta_bytes=822368 combine_payload_bytes=919371776 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=920194144 ingress_bytes=1271693312 link_bytes=1271693312 token_bytes=1839565920 weight_movement_bytes=352321536 total_nvlink_bytes=2191887456 weight_to_token_ratio=0.191524 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.120951716e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=913047552 dispatch_meta_bytes=885976 combine_payload_bytes=913047552 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=913933528 ingress_bytes=1529610240 link_bytes=1529610240 token_bytes=1826981080 weight_movement_bytes=616562688 total_nvlink_bytes=2443543768 weight_to_token_ratio=0.337476 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.565178853e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=896888 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940207992 ingress_bytes=1379713024 link_bytes=1379713024 token_bytes=1879519096 weight_movement_bytes=440401920 total_nvlink_bytes=2319921016 weight_to_token_ratio=0.234316 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=28416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725817014e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128274 payload_rows_remote=111890 dup_rows=133870 dup_rows_remote=116076 dedup_ratio=0.510674 router_tokens_per_expert=[2, 621, 39, 3, 63, 5, 0, 44, 0, 8152, 0, 2, 173, 7963, 8179, 13, 35, 8167, 16, 9, 8186, 2, 0, 33, 8123, 9, 31, 0, 6, 8145, 31, 0, 8136, 90, 2, 9, 5984, 1830, 5, 6, 8085, 7540, 1, 57, 7, 46, 2, 6, 30, 6808, 8156, 3, 21, 3660, 32, 3, 8176, 46, 1759, 23, 4374, 0, 1, 8122] payload_rows_per_dst=[16384, 16378, 16384, 16366, 16384, 16384, 13616, 16378] dup_rows_per_dst=[17794, 16314, 16512, 16324, 22856, 31416, 382, 12272] combine_rows_per_src=[16384, 16378, 16384, 16366, 16384, 16384, 13616, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128812 payload_rows_remote=112428 dup_rows=133332 dup_rows_remote=116996 dedup_ratio=0.508621 router_tokens_per_expert=[8165, 7856, 1, 26, 3, 8191, 5, 11, 0, 8169, 0, 319, 0, 16, 8158, 0, 0, 5478, 13, 6432, 8141, 8167, 2333, 0, 8192, 17, 0, 1, 0, 2, 8183, 0, 44, 5619, 4479, 12, 38, 8, 0, 1, 8, 3, 8165, 0, 8026, 147, 11, 0, 167, 0, 0, 0, 0, 48, 11, 0, 0, 0, 8188, 0, 4, 8189, 25, 0] payload_rows_per_dst=[16368, 16372, 16384, 16384, 14192, 16384, 16344, 16384] dup_rows_per_dst=[10064, 16952, 28510, 16406, 11912, 16336, 16724, 16428] combine_rows_per_src=[16368, 16372, 16384, 16384, 14192, 16384, 16344, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=10 unused_prefetch_slots=22 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=5 remote_demand_peak=3 slot_utilization=0.312500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 10, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129074 payload_rows_remote=112690 dup_rows=133070 dup_rows_remote=109944 dedup_ratio=0.507622 router_tokens_per_expert=[0, 2, 28, 0, 7, 10, 0, 1, 57, 8183, 0, 0, 1316, 7444, 8181, 15, 3, 8162, 0, 1, 8187, 1, 25, 4, 8175, 0, 225, 0, 0, 8090, 12, 0, 8139, 0, 1, 0, 7856, 3749, 0, 10, 4747, 6765, 382, 0, 15, 29, 1, 0, 528, 7583, 8187, 2, 0, 3812, 2, 0, 8189, 7, 441, 0, 4438, 0, 8, 8052] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 14390, 16384, 16380] dup_rows_per_dst=[16428, 17670, 16382, 16620, 23126, 9488, 23844, 9512] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 14390, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130056 payload_rows_remote=113678 dup_rows=132088 dup_rows_remote=114812 dedup_ratio=0.503876 router_tokens_per_expert=[8160, 8110, 0, 2, 0, 8192, 0, 18, 1, 8191, 0, 15, 3, 0, 8112, 2, 24, 7742, 6, 210, 8078, 8142, 2324, 3, 8191, 3, 5, 8, 8, 0, 8186, 0, 46, 7598, 5677, 14, 4, 6, 0, 1, 8, 0, 7175, 40, 8189, 320, 0, 13, 1459, 5, 0, 5, 0, 345, 0, 0, 19, 1, 8172, 22, 0, 8184, 33, 0] payload_rows_per_dst=[16384, 16384, 16378, 16384, 15382, 16384, 16384, 16376] dup_rows_per_dst=[32580, 16264, 17276, 16418, 11310, 15106, 6648, 16486] combine_rows_per_src=[16384, 16384, 16378, 16384, 15382, 16384, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=916602880 dispatch_meta_bytes=911864 combine_payload_bytes=916602880 prefetch_bytes=440401920 reduce_bytes=0 egress_bytes=917514744 ingress_bytes=1357004800 link_bytes=1357004800 token_bytes=1834117624 weight_movement_bytes=440401920 total_nvlink_bytes=2274519544 weight_to_token_ratio=0.240117 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2921 gemm_tail_pad_fraction=0.017045 gemm_active_groups=23 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.598660438e-05 prefetch_bytes_per_gemm_flop=2.466066919e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120410 payload_rows_remote=104026 dup_rows=141734 dup_rows_remote=125318 dedup_ratio=0.540672 router_tokens_per_expert=[8135, 8175, 0, 0, 0, 8175, 3, 8, 2, 8174, 2, 363, 7, 0, 8116, 16, 10, 5093, 1, 17, 8103, 8182, 7, 16, 8172, 1, 11, 3, 0, 0, 8176, 6, 2, 7605, 5295, 0, 78, 1, 0, 2, 0, 11, 7439, 10, 8144, 64, 1, 3, 6955, 43, 14, 0, 0, 1, 54, 18, 0, 7, 8151, 5, 2, 8190, 0, 3] payload_rows_per_dst=[5740, 16384, 16380, 16384, 16372, 16384, 16382, 16384] dup_rows_per_dst=[4, 16890, 10258, 16356, 25864, 25588, 30358, 16416] combine_rows_per_src=[5740, 16384, 16380, 16384, 16372, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131010 payload_rows_remote=114626 dup_rows=131134 dup_rows_remote=114790 dedup_ratio=0.500237 router_tokens_per_expert=[8176, 6329, 1, 3, 0, 8192, 1, 0, 4, 8177, 3, 64, 0, 1, 8153, 0, 0, 6763, 0, 1325, 8186, 8192, 957, 9, 8169, 0, 2, 0, 2, 2, 8189, 0, 0, 8180, 6664, 18, 17, 1, 0, 10, 0, 1, 5646, 21, 8135, 113, 9, 2, 4951, 9, 2, 0, 0, 5, 0, 0, 1, 0, 8188, 0, 0, 8189, 0, 10] payload_rows_per_dst=[16384, 16380, 16358, 16384, 16384, 16354, 16382, 16384] dup_rows_per_dst=[18370, 16424, 14278, 16344, 13396, 11500, 24430, 16392] combine_rows_per_src=[16384, 16380, 16358, 16384, 16384, 16354, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130746 payload_rows_remote=114362 dup_rows=131398 dup_rows_remote=114946 dedup_ratio=0.501244 router_tokens_per_expert=[8177, 8043, 0, 41, 0, 8186, 0, 50, 0, 8190, 1, 38, 0, 6, 8182, 1, 23, 6826, 22, 102, 8192, 8049, 5, 48, 8191, 0, 7, 1, 0, 0, 8183, 0, 10, 7932, 6506, 0, 0, 0, 10, 0, 0, 0, 7519, 9, 8185, 576, 0, 1, 3015, 12, 0, 0, 0, 331, 1, 10, 0, 1, 8188, 8, 1, 8191, 2, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16078, 16384, 16364, 16384] dup_rows_per_dst=[32610, 16452, 13860, 16380, 12838, 16196, 6664, 16398] combine_rows_per_src=[16384, 16384, 16384, 16384, 16078, 16384, 16364, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=921010176 dispatch_meta_bytes=917696 combine_payload_bytes=921010176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=921927872 ingress_bytes=1097170944 link_bytes=1097170944 token_bytes=1842938048 weight_movement_bytes=176160768 total_nvlink_bytes=2019098816 weight_to_token_ratio=0.095587 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.143699305e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131018 payload_rows_remote=114634 dup_rows=131126 dup_rows_remote=113496 dedup_ratio=0.500206 router_tokens_per_expert=[8189, 8154, 0, 15, 0, 8192, 2, 3, 8, 8191, 0, 0, 1, 1, 8192, 0, 0, 7923, 2, 503, 8185, 8186, 5, 0, 8190, 3, 1, 0, 0, 7, 8188, 0, 0, 8189, 7359, 6, 1, 0, 0, 1, 0, 0, 8114, 5, 8176, 26, 1, 0, 628, 0, 0, 0, 5, 3, 0, 0, 0, 0, 8184, 0, 0, 8191, 0, 42] payload_rows_per_dst=[16336, 16384, 16384, 16384, 16384, 16378, 16384, 16384] dup_rows_per_dst=[12, 16402, 16866, 16394, 31106, 16266, 17630, 16450] combine_rows_per_src=[16336, 16384, 16384, 16384, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=923156480 dispatch_meta_bytes=890536 combine_payload_bytes=923156480 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=924047016 ingress_bytes=1275478016 link_bytes=1275478016 token_bytes=1847203496 weight_movement_bytes=352321536 total_nvlink_bytes=2199525032 weight_to_token_ratio=0.190732 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.142144479e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=931250176 dispatch_meta_bytes=913960 combine_payload_bytes=931250176 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=932164136 ingress_bytes=1459732480 link_bytes=1459732480 token_bytes=1863414312 weight_movement_bytes=528482304 total_nvlink_bytes=2391896616 weight_to_token_ratio=0.283610 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=28160 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.173892566e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=852180992 dispatch_meta_bytes=917376 combine_payload_bytes=852180992 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=853098368 ingress_bytes=1028341760 link_bytes=1028341760 token_bytes=1705279360 weight_movement_bytes=176160768 total_nvlink_bytes=1881440128 weight_to_token_ratio=0.103303 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.758284605e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939016192 dispatch_meta_bytes=917664 combine_payload_bytes=939016192 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=939933856 ingress_bytes=1115176960 link_bytes=1115176960 token_bytes=1878950048 weight_movement_bytes=176160768 total_nvlink_bytes=2055110816 weight_to_token_ratio=0.093755 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244525478e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=936853504 dispatch_meta_bytes=917232 combine_payload_bytes=936853504 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=937770736 ingress_bytes=1289175040 link_bytes=1289175040 token_bytes=1874624240 weight_movement_bytes=352321536 total_nvlink_bytes=2226945776 weight_to_token_ratio=0.187942 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.218842096e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939081728 dispatch_meta_bytes=912520 combine_payload_bytes=939081728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939994248 ingress_bytes=1291403264 link_bytes=1291403264 token_bytes=1879075976 weight_movement_bytes=352321536 total_nvlink_bytes=2231397512 weight_to_token_ratio=0.187497 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017992 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231319220e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130894 payload_rows_remote=114510 dup_rows=131250 dup_rows_remote=99010 dedup_ratio=0.500679 router_tokens_per_expert=[8139, 7255, 1, 11, 5, 8141, 54, 13, 61, 8148, 17, 9, 25, 132, 8019, 11, 1, 6519, 0, 189, 8152, 8158, 67, 19, 8179, 28, 1, 2, 6, 9, 8164, 19, 0, 8107, 5946, 0, 247, 67, 8, 0, 0, 18, 6971, 0, 8151, 1, 0, 14, 5145, 17, 4, 0, 29, 319, 89, 0, 7, 0, 8112, 14, 19, 8172, 20, 41] payload_rows_per_dst=[16384, 16324, 16326, 16372, 16384, 16348, 16374, 16382] dup_rows_per_dst=[32240, 15412, 13586, 16444, 12366, 13962, 11130, 16110] combine_rows_per_src=[16384, 16324, 16326, 16372, 16384, 16348, 16374, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131018 payload_rows_remote=114634 dup_rows=131126 dup_rows_remote=114570 dedup_ratio=0.500206 router_tokens_per_expert=[0, 1380, 7623, 1708, 0, 2, 0, 0, 0, 1, 0, 51, 8190, 0, 1, 8153, 8148, 8174, 3, 0, 14, 73, 7780, 0, 0, 2, 29, 0, 44, 0, 8191, 0, 0, 8189, 8191, 8180, 2, 0, 116, 8192, 22, 0, 8043, 0, 263, 0, 0, 3, 1, 39, 0, 1, 90, 0, 0, 2, 0, 8144, 1, 6373, 7468, 0, 8185, 0] payload_rows_per_dst=[16372, 16382, 16348, 16382, 16384, 16384, 16384, 16382] dup_rows_per_dst=[5054, 16410, 15688, 16498, 16612, 16556, 16626, 27682] combine_rows_per_src=[16372, 16382, 16348, 16382, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938065920 dispatch_meta_bytes=854080 combine_payload_bytes=938065920 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=938920000 ingress_bytes=1642708992 link_bytes=1642708992 token_bytes=1876985920 weight_movement_bytes=704643072 total_nvlink_bytes=2581628992 weight_to_token_ratio=0.375412 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=29184 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.198484655e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126730 payload_rows_remote=110346 dup_rows=135414 dup_rows_remote=120826 dedup_ratio=0.516563 router_tokens_per_expert=[3, 1978, 7908, 4218, 0, 0, 0, 14, 1, 12, 120, 0, 8184, 0, 0, 8189, 8179, 8183, 0, 0, 32, 62, 7208, 5, 2, 1, 684, 0, 0, 11, 8180, 0, 0, 8077, 8174, 8182, 16, 0, 29, 8158, 25, 0, 7983, 3, 55, 0, 2, 0, 0, 5, 0, 26, 522, 1, 0, 0, 0, 8127, 1, 2202, 8168, 0, 8142, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16378, 16384, 12052] dup_rows_per_dst=[24244, 16632, 14588, 17738, 16206, 24232, 17406, 4368] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16378, 16384, 12052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130374 payload_rows_remote=113990 dup_rows=131770 dup_rows_remote=100048 dedup_ratio=0.502663 router_tokens_per_expert=[18, 0, 7916, 5195, 9, 24, 1, 0, 13, 0, 115, 2, 8177, 0, 0, 8183, 8156, 8173, 0, 3, 24, 124, 8120, 2, 0, 2, 0, 0, 0, 0, 8001, 1, 389, 8170, 8069, 8153, 0, 10, 473, 8171, 0, 4, 5461, 1, 46, 2, 2, 13, 0, 170, 0, 124, 15, 12, 2, 22, 38, 8176, 13, 7626, 5599, 1, 8028, 23] payload_rows_per_dst=[16044, 16384, 16374, 16350, 16352, 16102, 16384, 16384] dup_rows_per_dst=[10282, 15636, 16484, 16004, 13632, 11012, 16998, 31722] combine_rows_per_src=[16044, 16384, 16374, 16350, 16352, 16102, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=115298 dedup_ratio=0.500069 router_tokens_per_expert=[8180, 7959, 0, 0, 0, 8191, 1, 2, 12, 8165, 0, 0, 2, 0, 8186, 3, 0, 6270, 6, 1332, 8190, 8180, 634, 25, 8179, 0, 0, 0, 0, 0, 8184, 0, 0, 8173, 7908, 3, 1, 0, 3, 0, 0, 6, 8093, 0, 8184, 3, 0, 1, 576, 8, 10, 0, 0, 0, 5, 4, 0, 0, 8184, 1, 7, 8190, 3, 8] payload_rows_per_dst=[16382, 16384, 16384, 16368, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15902, 16352, 16532, 16358, 15792, 16190, 17562, 16402] combine_rows_per_src=[16382, 16384, 16384, 16368, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114590 dedup_ratio=0.500069 router_tokens_per_expert=[1, 40, 7283, 3636, 0, 0, 7, 0, 2, 17, 14, 6, 8186, 0, 3, 8185, 8191, 8191, 2, 0, 0, 135, 8182, 0, 0, 1, 24, 0, 36, 0, 8190, 0, 5, 8153, 8186, 8171, 6, 0, 1, 8153, 0, 2, 6362, 3, 57, 1, 0, 0, 2, 40, 6, 26, 82, 0, 0, 0, 0, 8137, 3, 7067, 8088, 8, 8179, 2] payload_rows_per_dst=[16382, 16382, 16382, 16384, 16382, 16384, 16384, 16374] dup_rows_per_dst=[13740, 16444, 16638, 16500, 16360, 14286, 16536, 20586] combine_rows_per_src=[16382, 16382, 16382, 16384, 16382, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128772 payload_rows_remote=112388 dup_rows=133372 dup_rows_remote=116614 dedup_ratio=0.508774 router_tokens_per_expert=[0, 11, 8054, 6296, 4, 0, 0, 6, 0, 4, 81, 115, 8189, 0, 0, 8182, 8192, 8185, 0, 11, 15, 0, 7841, 2, 1, 2, 0, 1, 6, 0, 8182, 0, 0, 8189, 8190, 8185, 0, 0, 4, 8192, 0, 1, 3558, 2, 267, 0, 3, 3, 1, 83, 6, 23, 84, 0, 0, 0, 4, 8042, 0, 6525, 8125, 11, 8192, 2] payload_rows_per_dst=[16366, 16384, 16384, 16376, 16380, 16384, 16384, 14114] dup_rows_per_dst=[28428, 16758, 15738, 16378, 16380, 17740, 16770, 5180] combine_rows_per_src=[16366, 16384, 16384, 16376, 16380, 16384, 16384, 14114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939081728 dispatch_meta_bytes=916816 combine_payload_bytes=939081728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939998544 ingress_bytes=1291403264 link_bytes=1291403264 token_bytes=1879080272 weight_movement_bytes=352321536 total_nvlink_bytes=2231401808 weight_to_token_ratio=0.187497 gemm_rows_defined=132992 gemm_rows_charged=135168 gemm_tail_pad_rows=2176 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.016098 gemm_active_groups=22 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231319220e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=113738 dedup_ratio=0.500031 router_tokens_per_expert=[0, 22, 7784, 6348, 1, 0, 0, 0, 0, 0, 16, 13, 8192, 0, 0, 8186, 8191, 8192, 0, 4, 0, 5, 7631, 0, 0, 1, 22, 0, 36, 4, 8180, 0, 828, 8192, 8182, 8190, 0, 0, 282, 8192, 0, 5, 2202, 0, 17, 0, 0, 0, 0, 2, 0, 0, 478, 0, 0, 0, 0, 8183, 0, 7137, 8167, 0, 8187, 0] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[11932, 16430, 15278, 16486, 18582, 4440, 17342, 30590] combine_rows_per_src=[16378, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=903954432 dispatch_meta_bytes=924688 combine_payload_bytes=903954432 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=904879120 ingress_bytes=1256275968 link_bytes=1256275968 token_bytes=1808833552 weight_movement_bytes=352321536 total_nvlink_bytes=2161155088 weight_to_token_ratio=0.194778 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.034621025e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=933806080 dispatch_meta_bytes=856152 combine_payload_bytes=933806080 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=934662232 ingress_bytes=1638449152 link_bytes=1638449152 token_bytes=1868468312 weight_movement_bytes=704643072 total_nvlink_bytes=2573111384 weight_to_token_ratio=0.377123 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=26240 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.174631329e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=919872 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940296512 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879673152 weight_movement_bytes=176160768 total_nvlink_bytes=2055833920 weight_to_token_ratio=0.093719 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917040 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940293680 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879670320 weight_movement_bytes=176160768 total_nvlink_bytes=2055831088 weight_to_token_ratio=0.093719 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=920682496 dispatch_meta_bytes=916008 combine_payload_bytes=920682496 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=921598504 ingress_bytes=1096843264 link_bytes=1096843264 token_bytes=1842281000 weight_movement_bytes=176160768 total_nvlink_bytes=2018441768 weight_to_token_ratio=0.095621 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.141864433e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=913672 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940372232 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879830792 weight_movement_bytes=352321536 total_nvlink_bytes=2232152328 weight_to_token_ratio=0.187422 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.017992 gemm_active_groups=22 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125432 payload_rows_remote=109048 dup_rows=136712 dup_rows_remote=113096 dedup_ratio=0.521515 router_tokens_per_expert=[16, 1644, 4962, 5155, 50, 0, 4, 0, 29, 29, 8, 0, 8159, 32, 0, 8134, 7895, 8148, 23, 29, 11, 0, 8155, 9, 0, 8, 2, 0, 29, 14, 8128, 21, 25, 8157, 8142, 8060, 0, 5, 144, 8172, 45, 0, 6900, 9, 1, 11, 0, 7, 2, 0, 1, 110, 816, 0, 12, 42, 117, 8179, 12, 5284, 8164, 0, 7954, 7] payload_rows_per_dst=[16384, 16384, 16378, 16344, 16382, 16384, 16384, 10792] dup_rows_per_dst=[23616, 16398, 21916, 14982, 16956, 29818, 12978, 48] combine_rows_per_src=[16384, 16384, 16378, 16344, 16382, 16384, 16384, 10792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114668 dup_rows=131092 dup_rows_remote=114160 dedup_ratio=0.500076 router_tokens_per_expert=[11, 7995, 3, 0, 1522, 0, 3, 0, 7385, 44, 1, 8157, 8176, 18, 0, 0, 0, 0, 0, 14, 8186, 1, 1, 2, 7703, 5, 388, 8182, 8191, 1, 8143, 0, 10, 6762, 1, 0, 0, 0, 3, 0, 95, 128, 8188, 0, 0, 8192, 0, 55, 8153, 0, 8192, 8029, 0, 139, 6, 0, 0, 0, 0, 704, 8187, 95, 1, 0] payload_rows_per_dst=[16384, 16376, 16384, 16384, 16372, 16384, 16384, 16384] dup_rows_per_dst=[19046, 8782, 16330, 22148, 13532, 16932, 16348, 17974] combine_rows_per_src=[16384, 16376, 16384, 16384, 16372, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=893321216 dispatch_meta_bytes=888576 combine_payload_bytes=893321216 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=894209792 ingress_bytes=1069481984 link_bytes=1069481984 token_bytes=1787531008 weight_movement_bytes=176160768 total_nvlink_bytes=1963691776 weight_to_token_ratio=0.098550 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.988652687e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130964 payload_rows_remote=114580 dup_rows=131180 dup_rows_remote=114518 dedup_ratio=0.500412 router_tokens_per_expert=[34, 8167, 1, 0, 548, 5, 68, 2, 8039, 3377, 631, 8141, 8164, 2115, 0, 176, 3, 4, 0, 151, 8182, 0, 0, 5, 8093, 0, 23, 7935, 8192, 0, 8189, 1, 0, 5745, 0, 0, 0, 0, 1, 19, 0, 3, 8192, 0, 0, 8188, 0, 22, 8178, 0, 8071, 8149, 0, 6, 0, 0, 0, 25, 2, 203, 3982, 37, 0, 3] payload_rows_per_dst=[16364, 16384, 16384, 16384, 16366, 16384, 16314, 16384] dup_rows_per_dst=[17156, 28574, 16662, 16228, 11492, 16426, 16138, 8504] combine_rows_per_src=[16364, 16384, 16384, 16384, 16366, 16384, 16314, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114662 dup_rows=131104 dup_rows_remote=116632 dedup_ratio=0.500122 router_tokens_per_expert=[6, 7359, 2, 0, 3999, 0, 1169, 0, 7639, 609, 202, 8178, 8183, 268, 10, 7, 2, 2, 2, 42, 8191, 0, 0, 4, 8090, 0, 0, 8153, 8179, 1, 7223, 111, 0, 6880, 5, 0, 13, 0, 5, 15, 6, 0, 8186, 0, 2, 8183, 3, 8, 8185, 115, 8175, 8079, 1, 157, 4, 1, 0, 20, 0, 660, 4736, 0, 2, 0] payload_rows_per_dst=[16384, 16382, 16382, 16378, 16384, 16384, 16372, 16374] dup_rows_per_dst=[24996, 17444, 16474, 14472, 13816, 16392, 16692, 10818] combine_rows_per_src=[16384, 16382, 16382, 16378, 16384, 16384, 16372, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123128 payload_rows_remote=106750 dup_rows=139016 dup_rows_remote=108880 dedup_ratio=0.530304 router_tokens_per_expert=[58, 8068, 0, 5, 4, 1, 9, 0, 8169, 4100, 236, 8137, 8128, 117, 0, 0, 117, 13, 26, 22, 8146, 23, 0, 0, 5901, 1, 352, 8077, 7927, 0, 8155, 2, 1, 6377, 0, 2, 43, 2, 66, 0, 16, 92, 8163, 19, 0, 8134, 1, 8, 8140, 10, 8154, 7984, 0, 9, 19, 3, 0, 6, 0, 338, 7665, 3, 10, 13] payload_rows_per_dst=[16206, 8642, 16384, 16384, 16382, 16384, 16368, 16378] dup_rows_per_dst=[7992, 2148, 15018, 20186, 29162, 16028, 18346, 30136] combine_rows_per_src=[16206, 8642, 16384, 16384, 16382, 16384, 16368, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=106488 dedup_ratio=0.500038 router_tokens_per_expert=[0, 5012, 0, 5, 902, 0, 3, 0, 6426, 5280, 407, 8172, 8189, 24, 5, 175, 46, 5, 0, 240, 8191, 1, 1, 0, 8185, 0, 1, 8192, 8170, 0, 8189, 21, 1, 6417, 57, 0, 2, 0, 2, 0, 193, 1, 8190, 0, 0, 8191, 1, 0, 8180, 0, 8167, 7989, 2, 20, 20, 0, 0, 1, 0, 17, 7753, 26, 0, 0] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[11806, 24594, 16944, 16408, 12958, 16768, 16016, 15588] combine_rows_per_src=[16378, 16384, 16384, 16384, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130808 payload_rows_remote=114424 dup_rows=131336 dup_rows_remote=114952 dedup_ratio=0.501007 router_tokens_per_expert=[1, 221, 6484, 7232, 0, 15, 15, 6, 0, 14, 3, 0, 8188, 0, 0, 8173, 8059, 8174, 5, 2, 0, 96, 7428, 0, 0, 0, 10, 0, 91, 19, 8183, 0, 12, 8181, 8192, 8163, 0, 0, 0, 8186, 6, 0, 6231, 0, 398, 6, 0, 0, 0, 34, 0, 0, 10, 0, 6, 0, 0, 8187, 2, 5802, 7044, 0, 8191, 2] payload_rows_per_dst=[16372, 16384, 16132, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14166, 16372, 15048, 16570, 16384, 13278, 16416, 23102] combine_rows_per_src=[16372, 16384, 16132, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=915312 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940275568 ingress_bytes=1291681792 link_bytes=1291681792 token_bytes=1879635824 weight_movement_bytes=352321536 total_nvlink_bytes=2231957360 weight_to_token_ratio=0.187441 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232878861e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131006 payload_rows_remote=114622 dup_rows=131138 dup_rows_remote=114826 dedup_ratio=0.500252 router_tokens_per_expert=[0, 7795, 0, 0, 612, 0, 1, 0, 6748, 4363, 1, 8180, 8191, 11, 7, 1, 0, 0, 0, 45, 8192, 0, 0, 2, 6859, 5, 20, 8190, 8166, 9, 7919, 0, 0, 8188, 0, 0, 12, 0, 44, 0, 0, 5, 8192, 0, 0, 8189, 0, 40, 8188, 0, 8150, 8176, 0, 22, 0, 0, 0, 0, 1, 342, 6206, 0, 0, 0] payload_rows_per_dst=[16380, 16374, 16384, 16352, 16384, 16384, 16384, 16364] dup_rows_per_dst=[16814, 5118, 16470, 18280, 24350, 16468, 16312, 17326] combine_rows_per_src=[16380, 16374, 16384, 16352, 16384, 16384, 16384, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 2, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 4, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938639360 dispatch_meta_bytes=916392 combine_payload_bytes=938639360 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939555752 ingress_bytes=1114800128 link_bytes=1114800128 token_bytes=1878195112 weight_movement_bytes=176160768 total_nvlink_bytes=2054355880 weight_to_token_ratio=0.093793 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.242415376e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=925176 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940236280 ingress_bytes=1643954176 link_bytes=1643954176 token_bytes=1879547384 weight_movement_bytes=704643072 total_nvlink_bytes=2584190456 weight_to_token_ratio=0.374900 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205457165e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=874496000 dispatch_meta_bytes=862520 combine_payload_bytes=874496000 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=875358520 ingress_bytes=1226817536 link_bytes=1226817536 token_bytes=1749854520 weight_movement_bytes=352321536 total_nvlink_bytes=2102176056 weight_to_token_ratio=0.201343 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.018939 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.869666103e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=884664 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=940326840 ingress_bytes=1644085248 link_bytes=1644085248 token_bytes=1879769016 weight_movement_bytes=704643072 total_nvlink_bytes=2584412088 weight_to_token_ratio=0.374856 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=27264 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206191114e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=937361408 dispatch_meta_bytes=917504 combine_payload_bytes=937361408 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=938278912 ingress_bytes=1818165248 link_bytes=1818165248 token_bytes=1875640320 weight_movement_bytes=880803840 total_nvlink_bytes=2756444160 weight_to_token_ratio=0.469602 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018096645e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130894 payload_rows_remote=114510 dup_rows=131250 dup_rows_remote=114208 dedup_ratio=0.500679 router_tokens_per_expert=[67, 8179, 18, 0, 177, 25, 50, 18, 7025, 108, 336, 7969, 8150, 2942, 0, 136, 9, 38, 37, 52, 8151, 54, 34, 0, 6538, 2, 0, 8036, 8176, 0, 8146, 18, 27, 7760, 0, 24, 0, 12, 0, 0, 17, 1, 8179, 1, 3, 8160, 18, 36, 8141, 5, 7827, 8092, 0, 31, 0, 23, 31, 0, 7, 865, 7309, 0, 12, 0] payload_rows_per_dst=[16384, 16368, 16384, 16376, 16380, 16384, 16240, 16378] dup_rows_per_dst=[17042, 21996, 16648, 13032, 15292, 15682, 15148, 16410] combine_rows_per_src=[16384, 16368, 16384, 16376, 16380, 16384, 16240, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=917792 combine_payload_bytes=938983424 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=939901216 ingress_bytes=1291304960 link_bytes=1291304960 token_bytes=1878884640 weight_movement_bytes=352321536 total_nvlink_bytes=2231206176 weight_to_token_ratio=0.187516 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230768759e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938065920 dispatch_meta_bytes=914872 combine_payload_bytes=938065920 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938980792 ingress_bytes=1114226688 link_bytes=1114226688 token_bytes=1877046712 weight_movement_bytes=176160768 total_nvlink_bytes=2053207480 weight_to_token_ratio=0.093850 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239204352e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130976 payload_rows_remote=114600 dup_rows=131168 dup_rows_remote=118070 dedup_ratio=0.500366 router_tokens_per_expert=[8191, 0, 0, 0, 8185, 0, 0, 8058, 0, 2, 1, 0, 0, 0, 0, 0, 8185, 8187, 1, 36, 0, 6, 0, 3, 0, 5129, 10, 21, 0, 0, 0, 8052, 72, 0, 11, 1, 8146, 0, 8186, 7827, 113, 0, 8162, 6455, 8190, 0, 8171, 0, 2, 26, 3, 0, 7823, 7617, 0, 156, 85, 0, 0, 0, 59, 108, 5787, 5] payload_rows_per_dst=[16370, 16374, 16384, 16322, 16382, 16376, 16384, 16384] dup_rows_per_dst=[16116, 16340, 16452, 10102, 15812, 13098, 31162, 12086] combine_rows_per_src=[16370, 16374, 16384, 16322, 16382, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114682 dup_rows=131080 dup_rows_remote=119936 dedup_ratio=0.500031 router_tokens_per_expert=[8186, 5, 40, 0, 8179, 2, 3, 8180, 0, 42, 9, 0, 0, 2, 5, 10, 8181, 8097, 0, 2, 2, 6, 0, 13, 0, 4549, 0, 1030, 7, 0, 0, 8177, 0, 0, 2, 14, 8190, 301, 8185, 7271, 0, 1, 8144, 8186, 8187, 0, 8180, 0, 13, 0, 0, 10, 2622, 8113, 0, 3, 2, 1, 6, 0, 39, 702, 8173, 0] payload_rows_per_dst=[16384, 16384, 16382, 16382, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16434, 16490, 16220, 11144, 23354, 16274, 13330, 17834] combine_rows_per_src=[16384, 16384, 16382, 16382, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130902 payload_rows_remote=114520 dup_rows=131242 dup_rows_remote=105592 dedup_ratio=0.500648 router_tokens_per_expert=[8169, 0, 27, 1, 8168, 0, 340, 8098, 4, 0, 38, 14, 6, 26, 1, 13, 8170, 8165, 36, 34, 0, 0, 10, 0, 1, 5214, 15, 3851, 24, 13, 1, 8183, 83, 4, 0, 3, 8149, 1, 8091, 8009, 9, 20, 8142, 7746, 8111, 0, 8180, 8, 7, 0, 5, 0, 36, 7655, 0, 37, 0, 27, 0, 88, 149, 10, 7880, 0] payload_rows_per_dst=[16336, 16384, 16384, 16378, 16374, 16306, 16358, 16382] dup_rows_per_dst=[7546, 15126, 16306, 18226, 10968, 22000, 15420, 25650] combine_rows_per_src=[16336, 16384, 16384, 16378, 16374, 16306, 16358, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130890 payload_rows_remote=114506 dup_rows=131254 dup_rows_remote=114794 dedup_ratio=0.500694 router_tokens_per_expert=[8129, 0, 0, 0, 8174, 0, 132, 8182, 0, 459, 1, 0, 0, 0, 0, 16, 8191, 8192, 3, 2, 0, 0, 33, 1, 0, 4057, 0, 163, 1, 0, 0, 8097, 30, 0, 47, 0, 8189, 7, 8124, 6693, 106, 1, 8187, 7807, 8178, 0, 8192, 6, 0, 12, 6, 0, 7458, 7579, 6, 2, 0, 0, 0, 5, 0, 1698, 4906, 0] payload_rows_per_dst=[16368, 16384, 16384, 16342, 16384, 16384, 16368, 16276] dup_rows_per_dst=[16608, 17298, 16460, 12342, 25574, 15840, 13932, 13200] combine_rows_per_src=[16368, 16384, 16384, 16342, 16384, 16384, 16368, 16276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130542 payload_rows_remote=114158 dup_rows=131602 dup_rows_remote=115098 dedup_ratio=0.502022 router_tokens_per_expert=[8169, 41, 14, 3, 8192, 1, 0, 8192, 0, 59, 26, 1, 0, 0, 0, 8, 8187, 8163, 0, 0, 0, 0, 1, 0, 0, 3665, 0, 1725, 8, 4, 0, 7035, 0, 5, 65, 0, 8192, 0, 8171, 8187, 37, 0, 8156, 7945, 8190, 0, 8160, 0, 0, 24, 5, 0, 4076, 7895, 4, 0, 30, 0, 0, 0, 0, 2380, 6056, 0] payload_rows_per_dst=[16384, 16384, 16376, 16344, 16384, 16380, 15908, 16382] dup_rows_per_dst=[17524, 16504, 16326, 24872, 15492, 15896, 8100, 16888] combine_rows_per_src=[16384, 16384, 16376, 16344, 16384, 16380, 15908, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938803200 dispatch_meta_bytes=930680 combine_payload_bytes=938803200 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939733880 ingress_bytes=1467285504 link_bytes=1467285504 token_bytes=1878537080 weight_movement_bytes=528482304 total_nvlink_bytes=2407019384 weight_to_token_ratio=0.281327 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.216186347e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130820 payload_rows_remote=114436 dup_rows=131324 dup_rows_remote=120818 dedup_ratio=0.500961 router_tokens_per_expert=[31, 8107, 0, 0, 31, 0, 56, 11, 7722, 6394, 1, 8185, 8178, 986, 0, 1, 41, 7, 0, 77, 8177, 0, 0, 0, 5232, 0, 3, 8186, 8080, 0, 8123, 16, 0, 5241, 0, 7, 0, 8, 11, 0, 0, 5, 8192, 0, 0, 8183, 0, 0, 8154, 8, 7992, 8126, 0, 225, 0, 0, 0, 49, 1, 251, 6968, 0, 6, 0] payload_rows_per_dst=[16354, 16384, 16384, 16282, 16384, 16384, 16280, 16368] dup_rows_per_dst=[16458, 30194, 16528, 10466, 10506, 16376, 16422, 14374] combine_rows_per_src=[16354, 16384, 16384, 16282, 16384, 16384, 16280, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=103354 dedup_ratio=0.500031 router_tokens_per_expert=[8184, 11, 0, 0, 8179, 2, 0, 8189, 0, 2, 0, 0, 2, 0, 9, 0, 8185, 8162, 0, 8, 0, 0, 12, 11, 0, 4112, 0, 1523, 4, 0, 0, 8133, 0, 0, 23, 0, 8192, 0, 8184, 7432, 30, 0, 8169, 7820, 8184, 0, 8183, 3, 0, 1, 0, 0, 5878, 7969, 0, 15, 4, 0, 3, 1, 2, 1629, 4622, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16382, 16384, 16384, 16380] dup_rows_per_dst=[14312, 16376, 16372, 11160, 14896, 15660, 27726, 14578] combine_rows_per_src=[16382, 16384, 16384, 16384, 16382, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=938472 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940413416 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879888360 weight_movement_bytes=176160768 total_nvlink_bytes=2056049128 weight_to_token_ratio=0.093708 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938147840 dispatch_meta_bytes=880448 combine_payload_bytes=938147840 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939028288 ingress_bytes=1290469376 link_bytes=1290469376 token_bytes=1877176128 weight_movement_bytes=352321536 total_nvlink_bytes=2229497664 weight_to_token_ratio=0.187687 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.019886 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226089837e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131010 payload_rows_remote=114626 dup_rows=131134 dup_rows_remote=105582 dedup_ratio=0.500237 router_tokens_per_expert=[8178, 0, 23, 4, 8054, 1, 500, 8100, 13, 56, 2, 0, 0, 30, 0, 0, 8011, 8169, 66, 0, 6, 0, 94, 2, 9, 4159, 28, 1765, 51, 0, 3, 8152, 13, 34, 0, 0, 8135, 19, 8143, 8018, 165, 65, 8174, 8068, 8053, 0, 8155, 54, 37, 3, 0, 45, 2350, 8140, 0, 167, 0, 1, 0, 28, 99, 0, 7611, 19] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16322, 16384, 16384, 16384] dup_rows_per_dst=[25552, 16476, 16592, 27956, 7544, 16426, 5100, 15488] combine_rows_per_src=[16384, 16384, 16384, 16384, 16322, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=935182336 dispatch_meta_bytes=917024 combine_payload_bytes=935182336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936099360 ingress_bytes=1287503872 link_bytes=1287503872 token_bytes=1871281696 weight_movement_bytes=352321536 total_nvlink_bytes=2223603232 weight_to_token_ratio=0.188278 gemm_rows_defined=132992 gemm_rows_charged=135168 gemm_tail_pad_rows=2176 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.016098 gemm_active_groups=22 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.209484253e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938033152 dispatch_meta_bytes=917200 combine_payload_bytes=938033152 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=938950352 ingress_bytes=1114193920 link_bytes=1114193920 token_bytes=1876983504 weight_movement_bytes=176160768 total_nvlink_bytes=2053144272 weight_to_token_ratio=0.093853 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239020865e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=937459712 dispatch_meta_bytes=941016 combine_payload_bytes=937459712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938400728 ingress_bytes=1289781248 link_bytes=1289781248 token_bytes=1875860440 weight_movement_bytes=352321536 total_nvlink_bytes=2228181976 weight_to_token_ratio=0.187819 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.222236608e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=872136 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940330696 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879789256 weight_movement_bytes=176160768 total_nvlink_bytes=2055950024 weight_to_token_ratio=0.093713 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939016192 dispatch_meta_bytes=880832 combine_payload_bytes=939016192 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939897024 ingress_bytes=1467498496 link_bytes=1467498496 token_bytes=1878913216 weight_movement_bytes=528482304 total_nvlink_bytes=2407395520 weight_to_token_ratio=0.281270 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217379014e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114642 dup_rows=131120 dup_rows_remote=104040 dedup_ratio=0.500183 router_tokens_per_expert=[8192, 6, 0, 3851, 0, 0, 8191, 0, 4, 8082, 7846, 0, 0, 4054, 0, 0, 2, 8192, 8189, 0, 0, 11, 8, 0, 24, 3, 0, 1, 8190, 25, 103, 0, 423, 1031, 5, 0, 8188, 5, 0, 1780, 6008, 8188, 6, 6, 8173, 7621, 1, 0, 8186, 0, 0, 1, 0, 20, 8186, 1, 0, 0, 0, 8186, 18, 0, 46, 19] payload_rows_per_dst=[16384, 16338, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[15904, 15662, 16420, 16684, 6480, 27080, 16404, 16486] combine_rows_per_src=[16384, 16338, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129908 payload_rows_remote=113524 dup_rows=132236 dup_rows_remote=115856 dedup_ratio=0.504440 router_tokens_per_expert=[8176, 0, 0, 1870, 5, 8, 8147, 1, 3, 8178, 7001, 0, 1, 1534, 0, 2, 3, 8180, 8182, 0, 6, 114, 119, 2, 0, 0, 6, 4, 8183, 0, 0, 1, 6784, 6416, 1, 5, 8188, 0, 0, 66, 693, 8188, 50, 1, 8191, 8191, 0, 0, 7783, 2, 10, 3, 0, 0, 8187, 12, 0, 192, 72, 8185, 0, 0, 7, 119] payload_rows_per_dst=[16384, 15222, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17802, 12252, 16830, 16380, 26536, 9678, 15610, 17148] combine_rows_per_src=[16384, 15222, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124864 payload_rows_remote=108480 dup_rows=137280 dup_rows_remote=120730 dedup_ratio=0.523682 router_tokens_per_expert=[8163, 0, 4, 910, 1, 200, 8140, 30, 6, 8068, 7067, 32, 10, 7984, 0, 18, 2, 8156, 8166, 15, 0, 0, 1, 1, 9, 2, 0, 6, 8158, 27, 68, 3, 4007, 3643, 43, 4, 8168, 12, 14, 296, 1166, 8184, 0, 0, 8157, 7394, 0, 4, 8174, 37, 20, 13, 0, 0, 8148, 0, 31, 3, 0, 8159, 11, 2, 69, 66] payload_rows_per_dst=[10376, 16360, 16362, 16384, 16384, 16234, 16380, 16384] dup_rows_per_dst=[1830, 28460, 11270, 15718, 24094, 22166, 17192, 16550] combine_rows_per_src=[10376, 16360, 16362, 16384, 16384, 16234, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130162 payload_rows_remote=114470 dup_rows=131982 dup_rows_remote=115834 dedup_ratio=0.503471 router_tokens_per_expert=[8172, 0, 1, 763, 0, 45, 8170, 0, 4, 8183, 5378, 5, 1, 6436, 5, 0, 0, 8190, 8191, 2, 20, 133, 0, 0, 0, 0, 153, 0, 8192, 1, 32, 1, 1645, 4158, 0, 0, 8189, 0, 0, 4255, 1818, 8192, 0, 0, 8184, 7413, 27, 0, 8187, 0, 0, 1, 0, 1, 8191, 1, 0, 103, 400, 8191, 1, 0, 34, 3] payload_rows_per_dst=[16384, 15692, 16384, 16384, 16384, 16166, 16384, 16384] dup_rows_per_dst=[26104, 16148, 16688, 16754, 17010, 5456, 16378, 17444] combine_rows_per_src=[16384, 15692, 16384, 16384, 16384, 16166, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130614 payload_rows_remote=114230 dup_rows=131530 dup_rows_remote=114378 dedup_ratio=0.501747 router_tokens_per_expert=[8190, 5, 2, 2316, 0, 176, 8182, 3, 1, 7758, 7553, 0, 0, 4101, 0, 0, 0, 8190, 8192, 0, 2, 86, 298, 0, 3, 0, 0, 0, 8192, 3, 10, 0, 98, 3428, 14, 0, 8189, 0, 0, 3331, 3688, 8192, 0, 1, 8191, 8044, 37, 4, 8188, 0, 11, 0, 2, 4, 8185, 0, 0, 0, 5, 8162, 0, 0, 32, 3] payload_rows_per_dst=[16384, 16260, 16384, 16384, 16062, 16374, 16384, 16382] dup_rows_per_dst=[27670, 14374, 17152, 16416, 7752, 15366, 16396, 16404] combine_rows_per_src=[16384, 16260, 16384, 16384, 16062, 16374, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114678 dup_rows=131106 dup_rows_remote=113570 dedup_ratio=0.500130 router_tokens_per_expert=[8189, 16, 4, 6, 8175, 0, 361, 8160, 1, 2, 44, 1, 0, 0, 0, 11, 8192, 8167, 8, 0, 0, 0, 2, 0, 10, 5707, 0, 1994, 0, 0, 0, 8192, 0, 0, 0, 2, 8121, 19, 8158, 8033, 4, 0, 8148, 8024, 8177, 0, 8192, 0, 19, 0, 0, 0, 3918, 5158, 0, 122, 3, 0, 0, 33, 0, 179, 7520, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16360, 16384, 16374, 16384] dup_rows_per_dst=[17060, 16472, 16354, 15422, 17536, 15968, 16830, 15464] combine_rows_per_src=[16384, 16384, 16384, 16384, 16360, 16384, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=874728 combine_payload_bytes=939147264 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940021992 ingress_bytes=1467629568 link_bytes=1467629568 token_bytes=1879169256 weight_movement_bytes=528482304 total_nvlink_bytes=2407651560 weight_to_token_ratio=0.281232 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218112962e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131030 payload_rows_remote=114650 dup_rows=131114 dup_rows_remote=114746 dedup_ratio=0.500160 router_tokens_per_expert=[8182, 1, 65, 3195, 7, 8, 8192, 0, 3, 8190, 6764, 0, 0, 5701, 2, 0, 0, 8187, 8192, 0, 0, 1959, 0, 0, 0, 13, 1, 0, 8192, 0, 31, 0, 709, 3227, 0, 0, 8190, 0, 0, 16, 2383, 8191, 0, 0, 8192, 7968, 0, 1, 8186, 0, 0, 0, 0, 0, 8188, 0, 5, 0, 4, 8182, 0, 0, 742, 3] payload_rows_per_dst=[16384, 16350, 16384, 16384, 16380, 16384, 16380, 16384] dup_rows_per_dst=[12154, 16782, 20292, 16472, 10474, 20702, 16368, 17870] combine_rows_per_src=[16384, 16350, 16384, 16384, 16380, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=929988608 dispatch_meta_bytes=917520 combine_payload_bytes=929988608 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=930906128 ingress_bytes=1106149376 link_bytes=1106149376 token_bytes=1860894736 weight_movement_bytes=176160768 total_nvlink_bytes=2037055504 weight_to_token_ratio=0.094665 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.193974776e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 2, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130784 payload_rows_remote=114402 dup_rows=131360 dup_rows_remote=104648 dedup_ratio=0.501099 router_tokens_per_expert=[8111, 21, 0, 319, 3, 0, 8168, 0, 35, 7921, 6230, 17, 0, 2737, 13, 10, 0, 8168, 8161, 35, 11, 19, 0, 0, 0, 1, 36, 0, 8117, 39, 0, 0, 2225, 4364, 0, 17, 8104, 19, 24, 6036, 2411, 8111, 39, 1, 8159, 7724, 0, 0, 8170, 6, 4, 8, 0, 69, 8158, 47, 17, 116, 0, 8159, 124, 1, 665, 122] payload_rows_per_dst=[16382, 16186, 16384, 16352, 16382, 16330, 16384, 16384] dup_rows_per_dst=[26712, 11614, 14812, 16316, 25142, 7628, 16402, 12734] combine_rows_per_src=[16382, 16186, 16384, 16352, 16382, 16330, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=888668160 dispatch_meta_bytes=916840 combine_payload_bytes=888668160 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=889585000 ingress_bytes=1064828928 link_bytes=1064828928 token_bytes=1778253160 weight_movement_bytes=176160768 total_nvlink_bytes=1954413928 weight_to_token_ratio=0.099064 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.962597515e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937738240 dispatch_meta_bytes=921216 combine_payload_bytes=937738240 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938659456 ingress_bytes=1290059776 link_bytes=1290059776 token_bytes=1876397696 weight_movement_bytes=352321536 total_nvlink_bytes=2228719232 weight_to_token_ratio=0.187765 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=29440 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223796248e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=935772160 dispatch_meta_bytes=914432 combine_payload_bytes=935772160 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=936686592 ingress_bytes=1288093696 link_bytes=1288093696 token_bytes=1872458752 weight_movement_bytes=352321536 total_nvlink_bytes=2224780288 weight_to_token_ratio=0.188160 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.212787021e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=912992 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=940355168 ingress_bytes=1820246016 link_bytes=1820246016 token_bytes=1879797344 weight_movement_bytes=880803840 total_nvlink_bytes=2760601184 weight_to_token_ratio=0.468563 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019261788e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=917584 combine_payload_bytes=939212800 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940130384 ingress_bytes=1115373568 link_bytes=1115373568 token_bytes=1879343184 weight_movement_bytes=176160768 total_nvlink_bytes=2055503952 weight_to_token_ratio=0.093735 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245626401e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937181184 dispatch_meta_bytes=876200 combine_payload_bytes=937181184 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=938057384 ingress_bytes=1377583104 link_bytes=1377583104 token_bytes=1875238568 weight_movement_bytes=440401920 total_nvlink_bytes=2315640488 weight_to_token_ratio=0.234851 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.713890351e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=113574 dedup_ratio=0.500053 router_tokens_per_expert=[34, 0, 0, 6811, 0, 197, 528, 8152, 0, 0, 0, 0, 1050, 0, 20, 8192, 8183, 8122, 0, 0, 0, 76, 8, 0, 20, 7068, 7837, 7, 33, 5, 8172, 0, 0, 0, 8183, 0, 1237, 1, 0, 0, 577, 0, 10, 8169, 0, 2, 0, 0, 3, 0, 1, 8176, 0, 0, 17, 8192, 8190, 0, 8187, 12, 8192, 0, 7408, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16372, 16384, 16384, 16384] dup_rows_per_dst=[15062, 2140, 16394, 29900, 2470, 17512, 16394, 31214] combine_rows_per_src=[16382, 16384, 16384, 16384, 16372, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130936 payload_rows_remote=114644 dup_rows=131208 dup_rows_remote=114998 dedup_ratio=0.500519 router_tokens_per_expert=[1, 0, 0, 5691, 0, 10, 917, 8190, 1, 127, 22, 0, 5829, 0, 47, 8189, 8188, 8191, 2, 1, 5, 1, 12, 4, 1, 2285, 8141, 1, 0, 0, 5819, 4, 73, 0, 8167, 0, 3501, 4, 3, 5, 299, 0, 4, 8182, 0, 0, 2, 0, 1, 5, 0, 8186, 3, 29, 0, 8159, 8188, 8, 8192, 0, 8186, 6, 8175, 15] payload_rows_per_dst=[16380, 16382, 16384, 16292, 16384, 16378, 16384, 16352] dup_rows_per_dst=[13238, 27280, 16424, 16210, 20680, 16972, 16382, 4022] combine_rows_per_src=[16380, 16382, 16384, 16292, 16384, 16378, 16384, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131030 payload_rows_remote=114660 dup_rows=131114 dup_rows_remote=92680 dedup_ratio=0.500160 router_tokens_per_expert=[0, 0, 83, 8081, 12, 180, 0, 8185, 0, 51, 0, 20, 6075, 15, 164, 8162, 8135, 8023, 0, 0, 7, 8, 3, 23, 5, 3978, 7994, 1, 329, 0, 4741, 5, 34, 32, 8120, 8, 2437, 0, 4, 4, 60, 1, 70, 8159, 5, 10, 4, 2, 12, 13, 0, 8163, 7, 0, 5, 8136, 8136, 5, 8166, 13, 8178, 24, 6984, 0] payload_rows_per_dst=[16384, 16384, 16380, 16382, 16368, 16384, 16378, 16370] dup_rows_per_dst=[16698, 12590, 16018, 16300, 4910, 9870, 16294, 38434] combine_rows_per_src=[16384, 16384, 16380, 16382, 16368, 16384, 16378, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130900 payload_rows_remote=114518 dup_rows=131244 dup_rows_remote=111852 dedup_ratio=0.500656 router_tokens_per_expert=[34, 0, 0, 4155, 2, 1, 157, 8072, 1, 21, 1, 0, 1564, 0, 2, 8191, 8160, 4261, 0, 1, 0, 34, 0, 0, 1, 8015, 6760, 0, 134, 25, 6749, 0, 0, 0, 8144, 0, 8031, 16, 0, 4, 1344, 0, 3, 8169, 0, 1, 1, 0, 2, 0, 10, 8188, 7, 37, 4, 8169, 8189, 0, 8145, 0, 8116, 0, 8151, 0] payload_rows_per_dst=[16384, 16382, 16382, 16282, 16384, 16382, 16384, 16320] dup_rows_per_dst=[24462, 19392, 9670, 9942, 32296, 19032, 16450, 0] combine_rows_per_src=[16384, 16382, 16382, 16282, 16384, 16382, 16384, 16320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[2, 0, 2, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130230 payload_rows_remote=113846 dup_rows=131914 dup_rows_remote=117488 dedup_ratio=0.503212 router_tokens_per_expert=[13, 0, 26, 1321, 0, 8, 214, 7816, 0, 0, 26, 0, 1751, 0, 0, 8191, 8189, 7176, 0, 0, 0, 40, 0, 0, 0, 7870, 8103, 0, 10, 0, 7638, 0, 0, 0, 8184, 4, 6898, 0, 0, 0, 412, 0, 45, 8191, 0, 0, 0, 0, 1, 0, 2, 8163, 0, 0, 0, 8188, 8192, 18, 8173, 1, 8145, 0, 8063, 0] payload_rows_per_dst=[15858, 16384, 16384, 16384, 16384, 16384, 16384, 16068] dup_rows_per_dst=[4058, 19936, 14426, 29738, 30134, 17296, 16324, 2] combine_rows_per_src=[15858, 16384, 16384, 16384, 16384, 16384, 16384, 16068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=912992 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940322400 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879731808 weight_movement_bytes=176160768 total_nvlink_bytes=2055892576 weight_to_token_ratio=0.093716 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114668 dup_rows=131094 dup_rows_remote=117180 dedup_ratio=0.500084 router_tokens_per_expert=[8100, 8, 21, 406, 0, 0, 8187, 19, 0, 8166, 8004, 0, 0, 3750, 32, 12, 0, 8156, 8183, 0, 5, 540, 124, 0, 8, 0, 1, 3, 8179, 1, 4, 5, 3148, 2040, 0, 0, 8150, 0, 4, 1806, 5114, 8173, 0, 0, 8191, 7929, 0, 0, 8171, 0, 0, 2, 0, 0, 8184, 0, 1, 0, 52, 8181, 3, 0, 8, 1] payload_rows_per_dst=[16384, 16382, 16384, 16378, 16382, 16384, 16372, 16384] dup_rows_per_dst=[8906, 18062, 17632, 16370, 13914, 23430, 16342, 16438] combine_rows_per_src=[16384, 16382, 16384, 16378, 16382, 16384, 16372, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 2, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=918568 combine_payload_bytes=939163648 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940082216 ingress_bytes=1555726336 link_bytes=1555726336 token_bytes=1879245864 weight_movement_bytes=616562688 total_nvlink_bytes=2495808552 weight_to_token_ratio=0.328090 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=26624 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.711418089e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114678 dedup_ratio=0.500031 router_tokens_per_expert=[0, 1, 3, 8058, 0, 13, 108, 8115, 0, 0, 5, 0, 487, 1, 72, 8192, 8188, 8132, 0, 1, 0, 120, 0, 0, 0, 1265, 7657, 0, 7, 0, 8169, 0, 0, 0, 8184, 0, 7295, 0, 0, 0, 6, 0, 6, 8185, 1, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 2, 8192, 8192, 0, 8190, 0, 8087, 0, 7939, 0] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16212, 1130, 16498, 16768, 14574, 16396, 16402, 33100] combine_rows_per_src=[16384, 16384, 16384, 16378, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122300 payload_rows_remote=105918 dup_rows=139844 dup_rows_remote=110746 dedup_ratio=0.533463 router_tokens_per_expert=[29, 9, 314, 5835, 0, 81, 256, 8100, 0, 0, 113, 6, 8064, 2, 13, 8151, 8101, 6677, 67, 0, 48, 435, 47, 7, 0, 968, 7854, 0, 259, 8, 8138, 11, 0, 65, 8131, 2, 2021, 6, 13, 32, 324, 0, 17, 8146, 28, 54, 3, 0, 11, 23, 0, 8125, 22, 107, 121, 8028, 8151, 16, 7978, 1, 7941, 39, 8042, 32] payload_rows_per_dst=[16382, 16384, 16382, 8126, 16384, 16382, 16384, 15876] dup_rows_per_dst=[29098, 32546, 24454, 160, 20112, 17064, 16396, 14] combine_rows_per_src=[16382, 16384, 16382, 8126, 16384, 16382, 16384, 15876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=829360 combine_payload_bytes=939294720 prefetch_bytes=88080384 reduce_bytes=704643072 egress_bytes=940124080 ingress_bytes=1732018176 link_bytes=1732018176 token_bytes=1879418800 weight_movement_bytes=792723456 total_nvlink_bytes=2672142256 weight_to_token_ratio=0.421792 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.698578806e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938131456 dispatch_meta_bytes=905480 combine_payload_bytes=938131456 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939036936 ingress_bytes=1114292224 link_bytes=1114292224 token_bytes=1877168392 weight_movement_bytes=176160768 total_nvlink_bytes=2053329160 weight_to_token_ratio=0.093844 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239571326e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=932626432 dispatch_meta_bytes=925336 combine_payload_bytes=932626432 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=933551768 ingress_bytes=1108787200 link_bytes=1108787200 token_bytes=1866178200 weight_movement_bytes=176160768 total_nvlink_bytes=2042338968 weight_to_token_ratio=0.094397 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.208745489e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=927392 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=940287648 ingress_bytes=1203601408 link_bytes=1203601408 token_bytes=1879647904 weight_movement_bytes=264241152 total_nvlink_bytes=2143889056 weight_to_token_ratio=0.140580 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739665477e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917432 combine_payload_bytes=939458560 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940375992 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834552 weight_movement_bytes=176160768 total_nvlink_bytes=2055995320 weight_to_token_ratio=0.093711 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=867680256 dispatch_meta_bytes=866656 combine_payload_bytes=867680256 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=868546912 ingress_bytes=1043841024 link_bytes=1043841024 token_bytes=1736227168 weight_movement_bytes=176160768 total_nvlink_bytes=1912387936 weight_to_token_ratio=0.101462 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.845074014e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130928 payload_rows_remote=114562 dup_rows=131216 dup_rows_remote=117092 dedup_ratio=0.500549 router_tokens_per_expert=[4, 4, 0, 0, 8172, 8192, 1, 8192, 8189, 0, 0, 2, 8189, 2, 19, 165, 0, 8146, 4, 0, 0, 45, 0, 0, 2, 1065, 0, 284, 3, 0, 19, 1, 1, 284, 0, 6, 0, 3, 0, 8192, 9, 0, 8171, 8190, 0, 1, 7064, 0, 8188, 0, 0, 7390, 8169, 8183, 8130, 0, 59, 138, 0, 0, 8188, 6, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16366, 16260, 16384] dup_rows_per_dst=[16362, 16748, 16388, 16060, 16926, 14124, 17826, 16782] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16366, 16260, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131018 payload_rows_remote=114636 dup_rows=131126 dup_rows_remote=107406 dedup_ratio=0.500206 router_tokens_per_expert=[43, 0, 5, 175, 8190, 8176, 0, 8183, 8185, 2, 5, 5, 8178, 0, 156, 0, 6, 8189, 6, 5, 1, 0, 8, 12, 0, 3670, 0, 14, 1, 0, 0, 0, 0, 320, 1, 2, 1, 3, 1, 7767, 1, 0, 8148, 8184, 0, 22, 4450, 43, 8188, 0, 1, 8111, 8176, 8189, 8034, 0, 1, 0, 15, 0, 8162, 0, 16, 21] payload_rows_per_dst=[16384, 16382, 16384, 16382, 16376, 16378, 16348, 16384] dup_rows_per_dst=[16790, 16680, 16444, 23720, 16182, 8950, 15944, 16416] combine_rows_per_src=[16384, 16382, 16384, 16382, 16376, 16378, 16348, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131008 payload_rows_remote=114638 dup_rows=131136 dup_rows_remote=114152 dedup_ratio=0.500244 router_tokens_per_expert=[24, 36, 0, 2, 8088, 8174, 0, 8166, 8192, 15, 40, 1, 8140, 0, 620, 87, 41, 8008, 0, 0, 2, 52, 0, 0, 2, 3021, 0, 282, 4, 1, 26, 0, 2, 0, 0, 0, 0, 0, 3, 8175, 28, 0, 8157, 8181, 3, 861, 4113, 15, 8155, 5, 24, 7293, 8159, 8167, 8185, 20, 0, 307, 0, 4, 8163, 12, 12, 4] payload_rows_per_dst=[16384, 16384, 16382, 16374, 16358, 16384, 16372, 16370] dup_rows_per_dst=[12826, 10614, 14068, 10504, 14372, 21372, 30396, 16984] combine_rows_per_src=[16384, 16384, 16382, 16374, 16358, 16384, 16372, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938491904 dispatch_meta_bytes=926616 combine_payload_bytes=938491904 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939418520 ingress_bytes=1290813440 link_bytes=1290813440 token_bytes=1877910424 weight_movement_bytes=352321536 total_nvlink_bytes=2230231960 weight_to_token_ratio=0.187614 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228016452e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130854 payload_rows_remote=114470 dup_rows=131290 dup_rows_remote=114484 dedup_ratio=0.500832 router_tokens_per_expert=[4, 0, 0, 13, 8167, 8179, 0, 8186, 8190, 0, 0, 0, 8164, 0, 235, 6, 0, 8191, 0, 33, 3, 3, 2, 0, 0, 1805, 29, 11, 8, 0, 25, 0, 7, 129, 0, 0, 0, 0, 0, 7999, 0, 290, 8129, 8112, 1, 0, 6268, 1, 8168, 4, 0, 8141, 8147, 8191, 7948, 1, 40, 69, 0, 0, 8164, 9, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16262, 16290, 16384] dup_rows_per_dst=[16344, 16806, 16416, 20048, 16110, 21306, 7708, 16552] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16262, 16290, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130972 payload_rows_remote=114588 dup_rows=131172 dup_rows_remote=112402 dedup_ratio=0.500381 router_tokens_per_expert=[345, 1, 0, 4, 8070, 8192, 0, 8192, 8178, 0, 0, 0, 8166, 0, 0, 0, 0, 8192, 0, 1182, 0, 11, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 1, 181, 0, 0, 2, 0, 3, 8192, 0, 0, 8126, 8192, 0, 8, 5912, 12, 8192, 0, 0, 8069, 8189, 8182, 7920, 0, 5, 0, 18, 0, 8181, 0, 81, 0] payload_rows_per_dst=[16384, 16356, 16384, 16384, 16384, 16384, 16312, 16384] dup_rows_per_dst=[16840, 16332, 18770, 18504, 16758, 12722, 14676, 16570] combine_rows_per_src=[16384, 16356, 16384, 16384, 16384, 16384, 16312, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939098112 dispatch_meta_bytes=888168 combine_payload_bytes=939098112 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939986280 ingress_bytes=1291419648 link_bytes=1291419648 token_bytes=1879084392 weight_movement_bytes=352321536 total_nvlink_bytes=2231405928 weight_to_token_ratio=0.187496 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017992 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231410964e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130754 payload_rows_remote=114370 dup_rows=131390 dup_rows_remote=125466 dedup_ratio=0.501213 router_tokens_per_expert=[3, 0, 0, 6330, 0, 69, 0, 8179, 3, 49, 6, 0, 5682, 11, 5, 8183, 8185, 8176, 0, 15, 0, 414, 0, 0, 0, 2070, 8189, 0, 1, 34, 8145, 10, 0, 0, 8162, 2, 2614, 0, 0, 0, 96, 0, 206, 8191, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 51, 0, 8164, 8169, 8, 8192, 45, 7285, 0, 7932, 1] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16068] dup_rows_per_dst=[12780, 16062, 17196, 20514, 5924, 16940, 16436, 25538] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=915160 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940029656 ingress_bytes=1115275264 link_bytes=1115275264 token_bytes=1879144152 weight_movement_bytes=176160768 total_nvlink_bytes=2055304920 weight_to_token_ratio=0.093745 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245075940e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=106688 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 0, 0, 8181, 8192, 0, 8191, 8187, 0, 0, 0, 8191, 0, 1670, 5, 0, 8192, 0, 81, 12, 0, 0, 0, 3, 22, 0, 4, 0, 10, 0, 2, 0, 35, 0, 0, 0, 0, 0, 8187, 2, 1, 8189, 8191, 0, 58, 6383, 12, 8190, 0, 0, 8112, 8181, 8192, 8180, 0, 0, 2, 25, 0, 8189, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[16362, 19360, 16568, 8270, 16428, 13268, 24392, 16432] combine_rows_per_src=[16382, 16384, 16384, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128826 payload_rows_remote=112442 dup_rows=133318 dup_rows_remote=114212 dedup_ratio=0.508568 router_tokens_per_expert=[128, 32, 20, 0, 8100, 8150, 19, 8154, 8092, 29, 69, 0, 8051, 34, 1082, 0, 31, 8003, 0, 1, 7, 0, 6, 0, 0, 4, 0, 412, 41, 3, 0, 9, 177, 214, 2, 3, 45, 0, 19, 8135, 90, 0, 8000, 8143, 12, 12, 7746, 1, 8142, 131, 0, 6955, 8138, 8110, 8173, 12, 93, 52, 52, 114, 8023, 0, 0, 1] payload_rows_per_dst=[16384, 16376, 16340, 16384, 16384, 16384, 14262, 16312] dup_rows_per_dst=[19106, 18338, 16098, 17026, 17092, 23530, 7074, 15054] combine_rows_per_src=[16384, 16376, 16340, 16384, 16384, 16384, 14262, 16312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937738240 dispatch_meta_bytes=915816 combine_payload_bytes=937738240 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938654056 ingress_bytes=1290059776 link_bytes=1290059776 token_bytes=1876392296 weight_movement_bytes=352321536 total_nvlink_bytes=2228713832 weight_to_token_ratio=0.187765 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223796248e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938704896 dispatch_meta_bytes=907960 combine_payload_bytes=938704896 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=939612856 ingress_bytes=1202946048 link_bytes=1202946048 token_bytes=1878317752 weight_movement_bytes=264241152 total_nvlink_bytes=2142558904 weight_to_token_ratio=0.140680 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.735995734e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 4, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936919040 dispatch_meta_bytes=959344 combine_payload_bytes=936919040 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=937878384 ingress_bytes=1113079808 link_bytes=1113079808 token_bytes=1874797424 weight_movement_bytes=176160768 total_nvlink_bytes=2050958192 weight_to_token_ratio=0.093963 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.232782302e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=885472 combine_payload_bytes=939458560 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=940344032 ingress_bytes=1820262400 link_bytes=1820262400 token_bytes=1879802592 weight_movement_bytes=880803840 total_nvlink_bytes=2760606432 weight_to_token_ratio=0.468562 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019270963e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=921124864 dispatch_meta_bytes=906616 combine_payload_bytes=921124864 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=922031480 ingress_bytes=1361526784 link_bytes=1361526784 token_bytes=1843156344 weight_movement_bytes=440401920 total_nvlink_bytes=2283558264 weight_to_token_ratio=0.238939 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.623981661e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131036 payload_rows_remote=114652 dup_rows=131108 dup_rows_remote=111744 dedup_ratio=0.500137 router_tokens_per_expert=[7049, 205, 8140, 1, 5, 0, 0, 0, 0, 16, 0, 993, 8148, 8163, 8189, 51, 4915, 0, 62, 8192, 257, 488, 23, 2941, 6, 0, 4, 0, 0, 0, 1, 8120, 0, 8159, 0, 7307, 2, 0, 1, 0, 8116, 0, 15, 4, 0, 1715, 2, 11, 7070, 18, 29, 8192, 8190, 0, 0, 8080, 5, 0, 0, 0, 0, 3, 8184, 0] payload_rows_per_dst=[16382, 16384, 16384, 16382, 16352, 16384, 16384, 16384] dup_rows_per_dst=[14418, 17344, 18384, 16036, 14586, 19364, 14596, 16380] combine_rows_per_src=[16382, 16384, 16384, 16382, 16352, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=113158 dedup_ratio=0.500038 router_tokens_per_expert=[8034, 2, 8192, 4, 0, 0, 6, 12, 125, 38, 0, 5, 8152, 8191, 8190, 0, 3085, 2, 1389, 8178, 10, 76, 0, 1994, 41, 0, 0, 1, 0, 1725, 0, 7199, 1, 8189, 14, 8065, 44, 0, 2, 7, 8178, 0, 2, 0, 1, 446, 1, 0, 8149, 0, 96, 7986, 8185, 2, 0, 8174, 0, 372, 2, 0, 1, 325, 8176, 3] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16378, 16384, 16384] dup_rows_per_dst=[16116, 16636, 13088, 17924, 16260, 17220, 16082, 17756] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130926 payload_rows_remote=114542 dup_rows=131218 dup_rows_remote=114458 dedup_ratio=0.500557 router_tokens_per_expert=[7666, 2921, 8142, 20, 1, 0, 18, 26, 0, 0, 2, 1004, 8123, 8137, 8151, 0, 324, 0, 1062, 8176, 0, 219, 0, 1908, 2, 39, 15, 0, 11, 0, 11, 7957, 31, 8099, 47, 5742, 0, 34, 0, 0, 7870, 0, 0, 0, 22, 4610, 47, 268, 7846, 0, 8, 7754, 8170, 9, 0, 8161, 17, 3, 21, 20, 0, 82, 8187, 89] payload_rows_per_dst=[16384, 16318, 16380, 16384, 16350, 16356, 16370, 16384] dup_rows_per_dst=[18132, 9464, 15108, 13938, 15264, 9278, 33274, 16760] combine_rows_per_src=[16384, 16318, 16380, 16384, 16350, 16356, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 6, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130958 payload_rows_remote=114576 dup_rows=131186 dup_rows_remote=113296 dedup_ratio=0.500435 router_tokens_per_expert=[7833, 122, 8183, 0, 12, 0, 0, 4, 812, 155, 0, 29, 7975, 8165, 8192, 0, 3859, 14, 3939, 8188, 0, 0, 30, 773, 66, 1, 1, 0, 3, 78, 5, 7908, 6, 8188, 0, 7321, 114, 1, 0, 2, 8130, 0, 1, 8, 0, 565, 0, 5, 8120, 0, 65, 7651, 8149, 0, 0, 8189, 0, 48, 0, 0, 10, 0, 8152, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16272, 16384] dup_rows_per_dst=[15924, 17890, 17586, 16118, 15616, 17332, 14300, 16420] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16272, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=905584 combine_payload_bytes=939229184 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940134768 ingress_bytes=1115389952 link_bytes=1115389952 token_bytes=1879363952 weight_movement_bytes=176160768 total_nvlink_bytes=2055524720 weight_to_token_ratio=0.093734 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245718145e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=911344 combine_payload_bytes=939442176 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940353520 ingress_bytes=1203683328 link_bytes=1203683328 token_bytes=1879795696 weight_movement_bytes=264241152 total_nvlink_bytes=2144036848 weight_to_token_ratio=0.140569 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740124195e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129728 payload_rows_remote=113344 dup_rows=132416 dup_rows_remote=118120 dedup_ratio=0.505127 router_tokens_per_expert=[7993, 59, 8192, 0, 0, 0, 1, 0, 17, 0, 0, 1438, 7990, 8192, 8181, 0, 3420, 0, 0, 8192, 0, 139, 118, 3471, 27, 3, 0, 0, 0, 28, 0, 8164, 0, 8112, 0, 7714, 14, 0, 0, 0, 8054, 0, 0, 22, 0, 1007, 1, 8, 5972, 0, 0, 7680, 8069, 0, 0, 8190, 2, 824, 0, 0, 0, 1377, 8192, 209] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16258, 16192, 15360, 16384] dup_rows_per_dst=[16106, 18870, 14296, 16440, 15422, 18130, 11944, 21208] combine_rows_per_src=[16384, 16382, 16384, 16384, 16258, 16192, 15360, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938328064 dispatch_meta_bytes=916000 combine_payload_bytes=938328064 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939244064 ingress_bytes=1114488832 link_bytes=1114488832 token_bytes=1877572128 weight_movement_bytes=176160768 total_nvlink_bytes=2053732896 weight_to_token_ratio=0.093824 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.240672249e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131014 payload_rows_remote=114630 dup_rows=131130 dup_rows_remote=113886 dedup_ratio=0.500221 router_tokens_per_expert=[1, 19, 30, 21, 8153, 8169, 0, 8191, 8166, 0, 0, 6, 8122, 0, 13, 1, 0, 8171, 0, 11, 0, 0, 2, 0, 4, 47, 0, 243, 12, 0, 6, 0, 0, 445, 14, 0, 0, 5, 0, 8164, 0, 0, 8192, 8192, 0, 31, 7874, 68, 8180, 12, 0, 7768, 8186, 8176, 8170, 0, 0, 0, 16, 0, 8186, 0, 5, 0] payload_rows_per_dst=[16384, 16344, 16384, 16384, 16384, 16384, 16366, 16384] dup_rows_per_dst=[16446, 16272, 16368, 16952, 17244, 15946, 15534, 16368] combine_rows_per_src=[16384, 16344, 16384, 16384, 16384, 16384, 16366, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=100896 dedup_ratio=0.500053 router_tokens_per_expert=[7330, 9, 8181, 0, 0, 0, 0, 0, 515, 9, 0, 2515, 8192, 8187, 8192, 0, 2, 0, 1789, 8183, 6, 0, 6, 930, 0, 0, 0, 0, 0, 2, 1, 7770, 0, 8179, 0, 8178, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3622, 177, 0, 8102, 2, 1, 8192, 8190, 1, 0, 8192, 0, 8, 0, 0, 0, 1, 8191, 26] payload_rows_per_dst=[16372, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[14668, 14270, 13640, 15546, 16330, 10000, 30190, 16442] combine_rows_per_src=[16372, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126028 payload_rows_remote=109644 dup_rows=136116 dup_rows_remote=117688 dedup_ratio=0.519241 router_tokens_per_expert=[5940, 790, 7951, 2, 6, 101, 49, 0, 4, 9, 53, 165, 7278, 8113, 8138, 63, 3212, 264, 4594, 8136, 156, 62, 69, 1641, 22, 0, 13, 13, 30, 70, 1, 8163, 20, 8002, 0, 6831, 0, 36, 1, 12, 7823, 12, 0, 0, 19, 1809, 399, 154, 7987, 78, 0, 8063, 8181, 44, 18, 8168, 20, 0, 0, 171, 11, 0, 8105, 0] payload_rows_per_dst=[16384, 16216, 16384, 16384, 16096, 16382, 11838, 16344] dup_rows_per_dst=[18428, 20300, 27938, 18572, 21780, 20402, 36, 8660] combine_rows_per_src=[16384, 16216, 16384, 16384, 16096, 16382, 11838, 16344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938606592 dispatch_meta_bytes=911488 combine_payload_bytes=938606592 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=939518080 ingress_bytes=1555169280 link_bytes=1555169280 token_bytes=1878124672 weight_movement_bytes=616562688 total_nvlink_bytes=2494687360 weight_to_token_ratio=0.328286 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.708298808e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 3, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=928514048 dispatch_meta_bytes=925856 combine_payload_bytes=928514048 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=929439904 ingress_bytes=1104674816 link_bytes=1104674816 token_bytes=1857953952 weight_movement_bytes=176160768 total_nvlink_bytes=2034114720 weight_to_token_ratio=0.094814 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.185717856e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939048960 dispatch_meta_bytes=914064 combine_payload_bytes=939048960 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939963024 ingress_bytes=1115209728 link_bytes=1115209728 token_bytes=1879011984 weight_movement_bytes=176160768 total_nvlink_bytes=2055172752 weight_to_token_ratio=0.093752 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244708965e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=862280 combine_payload_bytes=939409408 prefetch_bytes=0 reduce_bytes=792723456 egress_bytes=940271688 ingress_bytes=1732132864 link_bytes=1732132864 token_bytes=1879681096 weight_movement_bytes=792723456 total_nvlink_bytes=2672404552 weight_to_token_ratio=0.421733 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699221011e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=898203648 dispatch_meta_bytes=909328 combine_payload_bytes=898203648 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=899112976 ingress_bytes=1162444800 link_bytes=1162444800 token_bytes=1797316624 weight_movement_bytes=264241152 total_nvlink_bytes=2061557776 weight_to_token_ratio=0.147020 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.509205652e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130992 payload_rows_remote=114608 dup_rows=131152 dup_rows_remote=111744 dedup_ratio=0.500305 router_tokens_per_expert=[0, 0, 0, 2, 6038, 6874, 0, 0, 0, 1187, 8157, 0, 2917, 0, 0, 9, 0, 19, 1, 0, 0, 8183, 0, 10, 7915, 0, 0, 0, 0, 26, 0, 8105, 2, 0, 646, 0, 1, 8166, 8192, 134, 8192, 0, 0, 0, 0, 0, 7054, 8192, 0, 0, 8188, 8188, 0, 0, 8192, 0, 0, 0, 0, 8159, 0, 131, 8192, 0] payload_rows_per_dst=[16374, 16384, 16384, 16314, 16384, 16384, 16384, 16384] dup_rows_per_dst=[12346, 8156, 16426, 15778, 17898, 19408, 24560, 16580] combine_rows_per_src=[16374, 16384, 16384, 16314, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114640 dup_rows=131122 dup_rows_remote=115242 dedup_ratio=0.500191 router_tokens_per_expert=[4, 3, 0, 0, 8131, 5759, 0, 2, 9, 639, 8188, 4, 5087, 16, 19, 0, 0, 0, 0, 0, 1, 8162, 6, 3, 8191, 0, 0, 3, 1, 54, 0, 7882, 0, 2, 219, 0, 0, 7674, 8179, 365, 8135, 35, 6, 0, 0, 0, 7943, 8186, 0, 0, 8190, 8186, 6, 0, 6212, 1, 1, 3, 46, 7310, 0, 77, 8132, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16336, 16384, 16384] dup_rows_per_dst=[21776, 11540, 16334, 15880, 16494, 15902, 18442, 14754] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16336, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128984 payload_rows_remote=112600 dup_rows=133160 dup_rows_remote=109254 dedup_ratio=0.507965 router_tokens_per_expert=[22, 5, 347, 0, 1104, 7965, 0, 0, 0, 60, 8182, 57, 8164, 0, 42, 5, 0, 2, 0, 0, 4, 8081, 10, 5, 8085, 0, 1, 3, 0, 50, 15, 8154, 0, 0, 625, 90, 2, 8191, 8108, 143, 8077, 2, 0, 0, 17, 1, 8136, 8180, 68, 30, 8167, 8151, 0, 28, 7671, 7, 1, 0, 8, 6808, 0, 0, 8179, 19] payload_rows_per_dst=[16384, 16384, 16380, 16384, 14380, 16314, 16374, 16384] dup_rows_per_dst=[2502, 16636, 15728, 15592, 7852, 29962, 20982, 23906] combine_rows_per_src=[16384, 16384, 16380, 16384, 14380, 16314, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126106 payload_rows_remote=109722 dup_rows=136038 dup_rows_remote=103396 dedup_ratio=0.518944 router_tokens_per_expert=[0, 0, 0, 0, 4276, 8186, 0, 0, 10, 0, 8190, 1, 8153, 0, 2, 3, 0, 0, 0, 0, 87, 8159, 0, 0, 8187, 59, 8, 0, 1, 363, 0, 8173, 5, 0, 50, 0, 0, 7712, 8192, 31, 8154, 3, 15, 0, 0, 0, 7728, 8173, 0, 2, 8190, 8192, 2, 0, 5695, 0, 0, 4, 3, 6835, 0, 26, 8178, 24] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16384, 16384, 11424, 16384] dup_rows_per_dst=[24892, 32642, 16488, 17198, 15596, 7300, 8166, 13756] combine_rows_per_src=[16378, 16384, 16384, 16384, 16384, 16384, 11424, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938868736 dispatch_meta_bytes=905408 combine_payload_bytes=938868736 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939774144 ingress_bytes=1467351040 link_bytes=1467351040 token_bytes=1878642880 weight_movement_bytes=528482304 total_nvlink_bytes=2407125184 weight_to_token_ratio=0.281311 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.216553322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=922419200 dispatch_meta_bytes=887416 combine_payload_bytes=922419200 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=923306616 ingress_bytes=1098579968 link_bytes=1098579968 token_bytes=1845725816 weight_movement_bytes=176160768 total_nvlink_bytes=2021886584 weight_to_token_ratio=0.095443 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.151589251e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=919528 combine_payload_bytes=939130880 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940050408 ingress_bytes=1115291648 link_bytes=1115291648 token_bytes=1879181288 weight_movement_bytes=176160768 total_nvlink_bytes=2055342056 weight_to_token_ratio=0.093743 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245167683e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114710 dedup_ratio=0.500061 router_tokens_per_expert=[0, 0, 0, 5, 7287, 7306, 0, 0, 5, 59, 8192, 0, 1468, 2, 0, 15, 0, 2, 0, 0, 0, 8190, 0, 0, 8192, 21, 0, 0, 0, 0, 0, 8149, 0, 0, 19, 0, 0, 8143, 8192, 274, 8192, 0, 0, 0, 0, 0, 8185, 8188, 0, 0, 8169, 8192, 0, 0, 8192, 0, 0, 0, 0, 8098, 0, 150, 8185, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16370, 16384, 16384] dup_rows_per_dst=[29150, 14908, 16378, 16340, 16872, 11732, 9226, 16482] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16370, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114668 dup_rows=131100 dup_rows_remote=114798 dedup_ratio=0.500107 router_tokens_per_expert=[4728, 3896, 8161, 8, 22, 0, 0, 24, 42, 0, 0, 42, 8192, 8170, 8184, 0, 1592, 35, 1488, 8187, 0, 0, 0, 2410, 8, 0, 4, 0, 4, 32, 35, 8178, 10, 8165, 0, 8163, 1, 0, 0, 0, 8165, 0, 0, 0, 0, 1640, 0, 0, 8182, 0, 0, 8183, 8175, 0, 0, 8181, 0, 567, 10, 0, 12, 4, 8161, 11] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16376, 16380, 16384, 16374] dup_rows_per_dst=[17300, 16536, 11040, 16500, 16302, 19580, 16346, 17496] combine_rows_per_src=[16378, 16384, 16384, 16384, 16376, 16380, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=106506 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 83, 0, 3761, 7756, 0, 0, 0, 2, 8192, 0, 5788, 2, 0, 0, 0, 0, 0, 1, 0, 8093, 0, 20, 8192, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 14, 4, 0, 8189, 8182, 56, 8192, 0, 0, 0, 0, 0, 8172, 8191, 5, 0, 8192, 8191, 0, 0, 8188, 1, 12, 0, 0, 7211, 0, 17, 8183, 0] payload_rows_per_dst=[16376, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[6824, 11584, 16226, 16364, 16510, 24536, 24578, 14462] combine_rows_per_src=[16376, 16384, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127736 payload_rows_remote=111352 dup_rows=134408 dup_rows_remote=109960 dedup_ratio=0.512726 router_tokens_per_expert=[18, 0, 442, 0, 6068, 5725, 26, 6, 9, 906, 8149, 42, 1034, 5, 0, 12, 20, 30, 26, 44, 13, 8133, 0, 148, 7998, 0, 47, 1, 23, 676, 6, 8145, 12, 5, 788, 2, 0, 8181, 7876, 2561, 8105, 0, 2, 8, 93, 28, 7355, 8144, 4, 23, 8126, 8072, 24, 49, 7824, 28, 0, 0, 0, 7414, 1, 431, 8163, 1] payload_rows_per_dst=[16384, 16344, 16384, 16382, 16382, 14896, 14622, 16342] dup_rows_per_dst=[24448, 20142, 17126, 17618, 24010, 7774, 7612, 15678] combine_rows_per_src=[16384, 16344, 16384, 16382, 16382, 14896, 14622, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=898842624 dispatch_meta_bytes=852472 combine_payload_bytes=898842624 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=899695096 ingress_bytes=1075003392 link_bytes=1075003392 token_bytes=1798537720 weight_movement_bytes=176160768 total_nvlink_bytes=1974698488 weight_to_token_ratio=0.097947 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.019570267e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917528 combine_payload_bytes=939393024 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940310552 ingress_bytes=1203634176 link_bytes=1203634176 token_bytes=1879703576 weight_movement_bytes=264241152 total_nvlink_bytes=2143944728 weight_to_token_ratio=0.140576 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739848964e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=917864 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940278120 ingress_bytes=1115521024 link_bytes=1115521024 token_bytes=1879638376 weight_movement_bytes=176160768 total_nvlink_bytes=2055799144 weight_to_token_ratio=0.093721 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246452093e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=884728 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940310520 ingress_bytes=1555988480 link_bytes=1555988480 token_bytes=1879736312 weight_movement_bytes=616562688 total_nvlink_bytes=2496299000 weight_to_token_ratio=0.328005 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.712885986e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114668 dup_rows=131102 dup_rows_remote=114188 dedup_ratio=0.500114 router_tokens_per_expert=[8174, 0, 384, 2, 8114, 38, 7810, 307, 0, 0, 0, 7990, 0, 0, 26, 4, 0, 0, 8185, 347, 2, 8190, 10, 7821, 8145, 0, 2, 2, 4, 8192, 0, 8190, 402, 4522, 8185, 0, 1, 1310, 8187, 0, 8186, 0, 0, 271, 0, 0, 0, 0, 47, 0, 0, 1625, 8192, 0, 83, 0, 5, 8109, 1, 0, 0, 0, 7, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16374, 16384, 16366] dup_rows_per_dst=[16928, 16040, 16356, 16302, 12456, 16914, 19880, 16226] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16374, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127228 payload_rows_remote=110844 dup_rows=134916 dup_rows_remote=117308 dedup_ratio=0.514664 router_tokens_per_expert=[8176, 0, 4579, 0, 8156, 23, 242, 701, 0, 3, 7, 7862, 21, 0, 0, 11, 0, 12, 8179, 0, 1, 8123, 21, 6995, 8154, 1, 0, 0, 21, 8165, 7, 8164, 1034, 4032, 8093, 0, 35, 0, 8129, 0, 7467, 13, 20, 7721, 2, 0, 3, 24, 13, 10, 0, 332, 8159, 0, 150, 7, 30, 8084, 0, 5, 0, 17, 67, 1] payload_rows_per_dst=[12614, 16380, 16352, 16348, 16382, 16384, 16384, 16384] dup_rows_per_dst=[2130, 19336, 10072, 16950, 21210, 30374, 17236, 17608] combine_rows_per_src=[12614, 16380, 16352, 16348, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=912195584 dispatch_meta_bytes=885248 combine_payload_bytes=912195584 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=913080832 ingress_bytes=1088356352 link_bytes=1088356352 token_bytes=1825276416 weight_movement_bytes=176160768 total_nvlink_bytes=2001437184 weight_to_token_ratio=0.096512 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=29184 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.094341269e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131026 payload_rows_remote=114646 dup_rows=131118 dup_rows_remote=113354 dedup_ratio=0.500175 router_tokens_per_expert=[8185, 3, 1826, 0, 8190, 26, 5803, 52, 1, 2, 8, 8109, 2, 0, 0, 0, 0, 2, 7879, 51, 2, 8184, 2, 6808, 8185, 0, 1, 0, 721, 8151, 3, 8190, 1240, 7879, 7214, 14, 0, 1623, 8177, 69, 7131, 123, 4, 968, 0, 0, 0, 0, 0, 2, 48, 0, 8000, 29, 0, 0, 0, 8137, 0, 0, 0, 10, 0, 18] payload_rows_per_dst=[16384, 16384, 16342, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15416, 16218, 13146, 17764, 19694, 16422, 16142, 16316] combine_rows_per_src=[16384, 16384, 16342, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=915424 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940275680 ingress_bytes=1115521024 link_bytes=1115521024 token_bytes=1879635936 weight_movement_bytes=176160768 total_nvlink_bytes=2055796704 weight_to_token_ratio=0.093721 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246452093e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129184 payload_rows_remote=112800 dup_rows=132960 dup_rows_remote=116744 dedup_ratio=0.507202 router_tokens_per_expert=[8187, 0, 4148, 117, 8173, 0, 4165, 0, 1, 11, 0, 8087, 14, 0, 0, 0, 0, 0, 8182, 11, 6, 8186, 14, 7508, 8192, 0, 4, 0, 115, 8177, 2, 8190, 391, 5013, 8188, 1, 63, 778, 8074, 213, 7199, 2, 2, 2255, 4, 0, 0, 0, 0, 4, 0, 1012, 8169, 7, 17, 0, 0, 8190, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 14498, 16382, 16384] dup_rows_per_dst=[16822, 16216, 16754, 16592, 28132, 4426, 17638, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 14498, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=908034048 dispatch_meta_bytes=912608 combine_payload_bytes=908034048 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=908946656 ingress_bytes=1260355584 link_bytes=1260355584 token_bytes=1816980704 weight_movement_bytes=352321536 total_nvlink_bytes=2169302240 weight_to_token_ratio=0.193905 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.057465172e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130136 payload_rows_remote=113752 dup_rows=132008 dup_rows_remote=115624 dedup_ratio=0.503571 router_tokens_per_expert=[8192, 10, 758, 2, 8181, 0, 8168, 262, 2, 0, 0, 7383, 0, 0, 0, 0, 0, 0, 8188, 43, 1, 8182, 15, 8133, 8108, 0, 18, 1, 620, 8192, 0, 8192, 573, 3891, 8192, 0, 2, 0, 8185, 7, 7730, 0, 0, 3034, 0, 0, 0, 0, 1, 0, 0, 459, 8171, 38, 0, 0, 0, 8133, 0, 0, 0, 0, 5, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 15460, 16372, 16384] dup_rows_per_dst=[18378, 14770, 16384, 17496, 13788, 17596, 17322, 16274] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 15460, 16372, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939180032 dispatch_meta_bytes=912000 combine_payload_bytes=939180032 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940092032 ingress_bytes=1467662336 link_bytes=1467662336 token_bytes=1879272064 weight_movement_bytes=528482304 total_nvlink_bytes=2407754368 weight_to_token_ratio=0.281216 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218296449e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129716 payload_rows_remote=113332 dup_rows=132428 dup_rows_remote=114578 dedup_ratio=0.505173 router_tokens_per_expert=[2, 13, 10, 0, 1200, 7446, 5, 0, 0, 2, 8179, 0, 7747, 0, 16, 0, 9, 0, 0, 0, 0, 8179, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 8192, 0, 5, 783, 0, 7, 8181, 8141, 0, 8139, 0, 26, 0, 9, 0, 8057, 8189, 0, 11, 8176, 8187, 0, 0, 8026, 7, 0, 13, 9, 7727, 0, 0, 8175, 16] payload_rows_per_dst=[15082, 16384, 16384, 16384, 16384, 16330, 16384, 16384] dup_rows_per_dst=[10462, 15504, 16370, 16376, 17850, 16132, 24238, 15496] combine_rows_per_src=[15082, 16384, 16384, 16384, 16384, 16330, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130486 payload_rows_remote=114102 dup_rows=131658 dup_rows_remote=114650 dedup_ratio=0.502235 router_tokens_per_expert=[8190, 0, 4069, 0, 8161, 8, 6357, 7, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 8184, 0, 0, 8192, 0, 5059, 8186, 0, 5, 0, 0, 8192, 0, 8192, 328, 5257, 8192, 0, 0, 0, 8189, 58, 8192, 0, 0, 3156, 0, 2, 0, 0, 0, 0, 0, 313, 8187, 0, 4, 0, 0, 8192, 0, 1, 0, 0, 3, 4] payload_rows_per_dst=[16338, 16384, 15844, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[20560, 16686, 8522, 16382, 13410, 22694, 17008, 16396] combine_rows_per_src=[16338, 16384, 15844, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=924057600 dispatch_meta_bytes=918176 combine_payload_bytes=924057600 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=924975776 ingress_bytes=1276379136 link_bytes=1276379136 token_bytes=1849033376 weight_movement_bytes=352321536 total_nvlink_bytes=2201354912 weight_to_token_ratio=0.190544 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.147190375e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129626 payload_rows_remote=113286 dup_rows=132518 dup_rows_remote=112402 dedup_ratio=0.505516 router_tokens_per_expert=[8029, 0, 8124, 0, 7883, 167, 1470, 5, 24, 0, 43, 8168, 58, 0, 41, 0, 5, 35, 8029, 296, 6, 8105, 0, 7765, 8117, 1, 0, 16, 0, 8018, 8, 8069, 832, 1211, 8148, 0, 186, 7, 8158, 0, 8122, 0, 41, 4280, 39, 15, 13, 34, 25, 0, 111, 1156, 8142, 22, 172, 0, 50, 7735, 10, 59, 22, 0, 0, 0] payload_rows_per_dst=[16340, 16384, 16374, 15036, 16370, 16384, 16384, 16354] dup_rows_per_dst=[20116, 19048, 34112, 13364, 12600, 8704, 8982, 15592] combine_rows_per_src=[16340, 16384, 16374, 15036, 16370, 16384, 16384, 16354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=931856384 dispatch_meta_bytes=917504 combine_payload_bytes=931856384 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=932773888 ingress_bytes=1372258304 link_bytes=1372258304 token_bytes=1864630272 weight_movement_bytes=440401920 total_nvlink_bytes=2305032192 weight_to_token_ratio=0.236187 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.684073694e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=928415744 dispatch_meta_bytes=911640 combine_payload_bytes=928415744 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=929327384 ingress_bytes=1456898048 link_bytes=1456898048 token_bytes=1857743128 weight_movement_bytes=528482304 total_nvlink_bytes=2386225432 weight_to_token_ratio=0.284475 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.158020930e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934723584 dispatch_meta_bytes=915008 combine_payload_bytes=934723584 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=935638592 ingress_bytes=1110884352 link_bytes=1110884352 token_bytes=1870362176 weight_movement_bytes=176160768 total_nvlink_bytes=2046522944 weight_to_token_ratio=0.094185 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.220488665e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130234 payload_rows_remote=114544 dup_rows=131910 dup_rows_remote=115812 dedup_ratio=0.503197 router_tokens_per_expert=[1, 8171, 0, 2, 8185, 0, 5, 0, 8191, 8039, 8184, 0, 8174, 0, 20, 54, 5, 0, 0, 1, 21, 453, 2491, 0, 1, 0, 3982, 0, 2, 0, 7896, 0, 0, 0, 8192, 1, 0, 0, 0, 2, 7897, 0, 0, 2, 5492, 2, 0, 6501, 147, 6264, 183, 5, 0, 12, 8190, 3, 7880, 21, 8, 0, 0, 10, 8190, 8192] payload_rows_per_dst=[16382, 16242, 16384, 16384, 16384, 15690, 16384, 16384] dup_rows_per_dst=[16346, 8168, 22292, 21124, 16386, 16098, 13224, 18272] combine_rows_per_src=[16382, 16242, 16384, 16384, 16384, 15690, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121144 payload_rows_remote=104798 dup_rows=141000 dup_rows_remote=119158 dedup_ratio=0.537872 router_tokens_per_expert=[0, 7070, 0, 0, 8137, 26, 40, 0, 8152, 7283, 8128, 0, 8176, 2, 317, 11, 72, 0, 5, 36, 62, 719, 4068, 1, 444, 8, 3680, 16, 0, 5, 7354, 2, 22, 0, 8172, 0, 0, 3, 0, 0, 8060, 25, 11, 37, 7036, 2, 19, 7241, 220, 8118, 94, 11, 14, 8, 8184, 143, 3460, 24, 13, 1, 1, 30, 8145, 8164] payload_rows_per_dst=[16356, 7084, 16384, 16050, 16360, 16180, 16384, 16346] dup_rows_per_dst=[14116, 36, 26180, 15062, 16336, 28696, 18732, 21842] combine_rows_per_src=[16356, 7084, 16384, 16050, 16360, 16180, 16384, 16346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=928038912 dispatch_meta_bytes=902752 combine_payload_bytes=928038912 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=928941664 ingress_bytes=1632681984 link_bytes=1632681984 token_bytes=1856980576 weight_movement_bytes=704643072 total_nvlink_bytes=2561623648 weight_to_token_ratio=0.379456 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.022727 gemm_active_groups=17 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.142337595e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114654 dup_rows=131106 dup_rows_remote=119634 dedup_ratio=0.500130 router_tokens_per_expert=[4, 8144, 0, 5, 8187, 7, 10, 5, 8109, 8192, 8165, 19, 8177, 0, 167, 26, 0, 0, 1, 0, 49, 70, 4, 0, 0, 0, 726, 0, 0, 11, 8182, 1, 0, 3, 8185, 0, 0, 0, 1, 0, 8160, 0, 3, 7, 7348, 70, 0, 8007, 0, 8186, 0, 1, 2, 3, 8186, 10, 8177, 0, 0, 1, 30, 67, 8182, 8182] payload_rows_per_dst=[16384, 16384, 16358, 16384, 16376, 16384, 16384, 16384] dup_rows_per_dst=[16340, 23910, 7420, 11472, 16366, 22676, 16392, 16530] combine_rows_per_src=[16384, 16384, 16358, 16384, 16376, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938344448 dispatch_meta_bytes=921424 combine_payload_bytes=938344448 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=939265872 ingress_bytes=1378746368 link_bytes=1378746368 token_bytes=1877610320 weight_movement_bytes=440401920 total_nvlink_bytes=2318012240 weight_to_token_ratio=0.234554 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=25472 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.720404144e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130896 payload_rows_remote=114512 dup_rows=131248 dup_rows_remote=98108 dedup_ratio=0.500671 router_tokens_per_expert=[5, 8175, 0, 0, 8192, 0, 1, 0, 8130, 8192, 8191, 0, 8168, 1, 0, 211, 0, 0, 1, 0, 0, 475, 3906, 0, 2, 0, 2284, 115, 7, 2, 8181, 0, 0, 0, 8185, 30, 0, 15, 2, 0, 8104, 5, 3, 17, 7457, 1, 0, 4557, 0, 7835, 81, 7, 0, 0, 8191, 2, 5947, 28, 10, 0, 1, 0, 8173, 8182] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16208, 16384, 16384] dup_rows_per_dst=[17650, 33140, 8642, 12814, 16444, 14776, 15848, 11934] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16208, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=858505216 dispatch_meta_bytes=895824 combine_payload_bytes=858505216 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=859401040 ingress_bytes=1298907136 link_bytes=1298907136 token_bytes=1717906256 weight_movement_bytes=440401920 total_nvlink_bytes=2158308176 weight_to_token_ratio=0.256360 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.273337771e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130206 payload_rows_remote=113822 dup_rows=131938 dup_rows_remote=128578 dedup_ratio=0.503304 router_tokens_per_expert=[0, 8192, 0, 3, 8154, 0, 0, 0, 8178, 7962, 8132, 0, 8192, 0, 225, 55, 0, 0, 0, 0, 425, 20, 1235, 0, 44, 0, 5420, 1, 0, 0, 8160, 0, 23, 0, 8192, 0, 32, 0, 0, 0, 8179, 0, 0, 0, 6371, 4, 0, 7670, 0, 7737, 90, 0, 0, 0, 8192, 0, 3650, 153, 0, 0, 2, 5, 8183, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 15518, 16384, 16384] dup_rows_per_dst=[16314, 32720, 3360, 19032, 16476, 20764, 15654, 7618] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 15518, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=937152 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940182720 ingress_bytes=1115406336 link_bytes=1115406336 token_bytes=1879428288 weight_movement_bytes=176160768 total_nvlink_bytes=2055589056 weight_to_token_ratio=0.093731 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245809888e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114656 dup_rows=131104 dup_rows_remote=121306 dedup_ratio=0.500122 router_tokens_per_expert=[8180, 0, 4659, 0, 8184, 0, 7641, 0, 0, 0, 15, 6354, 0, 32, 0, 0, 0, 0, 8156, 14, 32, 8178, 0, 8165, 8178, 17, 2, 14, 597, 8172, 13, 8176, 2985, 1879, 8170, 8, 45, 3, 8163, 1, 8140, 20, 4, 227, 6, 0, 0, 13, 1, 0, 0, 312, 8138, 1, 1, 0, 0, 8163, 0, 12, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16368, 16368, 16384] dup_rows_per_dst=[24584, 12762, 16350, 17610, 9798, 16778, 16894, 16328] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16368, 16368, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 4, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127144 payload_rows_remote=110760 dup_rows=135000 dup_rows_remote=115300 dedup_ratio=0.514984 router_tokens_per_expert=[5, 8191, 0, 0, 8192, 0, 2, 3, 8192, 8132, 8188, 0, 8188, 0, 5, 35, 0, 0, 0, 0, 0, 0, 4515, 0, 0, 0, 755, 0, 0, 0, 8187, 0, 0, 0, 8188, 9, 0, 0, 0, 0, 8081, 0, 0, 0, 4202, 0, 0, 7496, 61, 8183, 6, 0, 0, 0, 8192, 1, 7675, 0, 0, 0, 3, 1, 8192, 8192] payload_rows_per_dst=[16384, 13138, 16384, 16384, 16384, 15702, 16384, 16384] dup_rows_per_dst=[16402, 816, 25404, 9686, 16394, 23100, 19700, 23498] combine_rows_per_src=[16384, 13138, 16384, 16384, 16384, 15702, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938082304 dispatch_meta_bytes=850480 combine_payload_bytes=938082304 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=938932784 ingress_bytes=1818886144 link_bytes=1818886144 token_bytes=1877015088 weight_movement_bytes=880803840 total_nvlink_bytes=2757818928 weight_to_token_ratio=0.469258 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018500317e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125458 payload_rows_remote=109074 dup_rows=136686 dup_rows_remote=118816 dedup_ratio=0.521416 router_tokens_per_expert=[0, 8030, 4, 41, 8094, 0, 39, 99, 8173, 7560, 8128, 13, 8041, 0, 730, 384, 27, 8, 47, 129, 0, 20, 7244, 48, 17, 2, 6805, 0, 43, 45, 7983, 0, 5, 3, 8013, 1, 0, 0, 7, 0, 8165, 0, 22, 2, 116, 7, 52, 2235, 71, 7460, 404, 30, 14, 32, 8142, 431, 7708, 67, 14, 0, 1, 6, 8161, 8149] payload_rows_per_dst=[16384, 16384, 16382, 16382, 16350, 11206, 16374, 15996] dup_rows_per_dst=[17870, 34070, 14912, 21574, 16376, 186, 16150, 15548] combine_rows_per_src=[16384, 16384, 16382, 16382, 16350, 11206, 16374, 15996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=932429824 dispatch_meta_bytes=969600 combine_payload_bytes=932429824 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933399424 ingress_bytes=1284751360 link_bytes=1284751360 token_bytes=1865829248 weight_movement_bytes=352321536 total_nvlink_bytes=2218150784 weight_to_token_ratio=0.188828 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017992 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.194071334e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=943848 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940205800 ingress_bytes=1379663872 link_bytes=1379663872 token_bytes=1879467752 weight_movement_bytes=440401920 total_nvlink_bytes=2319869672 weight_to_token_ratio=0.234323 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725541783e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=907345920 dispatch_meta_bytes=904240 combine_payload_bytes=907345920 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=908250160 ingress_bytes=1171587072 link_bytes=1171587072 token_bytes=1815596080 weight_movement_bytes=264241152 total_nvlink_bytes=2079837232 weight_to_token_ratio=0.145540 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.560398559e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=893534208 dispatch_meta_bytes=911560 combine_payload_bytes=893534208 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=894445768 ingress_bytes=1069694976 link_bytes=1069694976 token_bytes=1787979976 weight_movement_bytes=176160768 total_nvlink_bytes=1964140744 weight_to_token_ratio=0.098525 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.989845353e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130770 payload_rows_remote=114386 dup_rows=131374 dup_rows_remote=90162 dedup_ratio=0.501152 router_tokens_per_expert=[0, 4, 0, 8149, 4, 0, 5, 405, 20, 1246, 4, 0, 8176, 0, 43, 0, 53, 8136, 7207, 49, 5, 0, 7, 18, 0, 0, 0, 52, 8173, 31, 241, 8169, 0, 8190, 0, 0, 8058, 8126, 7820, 0, 0, 33, 0, 0, 0, 8141, 0, 1, 21, 0, 2, 5, 146, 63, 10, 8088, 95, 7789, 19, 8185, 2, 8140, 71, 7870] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16118, 16348, 16384, 16384] dup_rows_per_dst=[15362, 2594, 14566, 12242, 12744, 16116, 16538, 41212] combine_rows_per_src=[16384, 16384, 16384, 16384, 16118, 16348, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127670 payload_rows_remote=111286 dup_rows=134474 dup_rows_remote=118142 dedup_ratio=0.512978 router_tokens_per_expert=[5, 2, 0, 8183, 0, 0, 33, 0, 0, 4115, 0, 2, 8189, 0, 11, 0, 1, 8168, 2988, 2, 0, 0, 12, 0, 0, 0, 1, 6, 8189, 0, 0, 8162, 0, 8178, 0, 7, 8169, 8185, 8186, 0, 0, 6, 17, 1, 0, 8188, 0, 3, 1, 2893, 0, 0, 0, 13, 1, 8192, 0, 8041, 0, 3314, 3, 8159, 3661, 7785] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16372, 16384, 16384, 12994] dup_rows_per_dst=[16410, 19830, 18988, 16332, 16352, 15624, 22186, 8752] combine_rows_per_src=[16384, 16384, 16384, 16384, 16372, 16384, 16384, 12994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130856 payload_rows_remote=114474 dup_rows=131288 dup_rows_remote=115090 dedup_ratio=0.500824 router_tokens_per_expert=[0, 0, 0, 8191, 4, 7, 0, 222, 0, 4430, 0, 0, 8181, 0, 1, 14, 4, 8190, 5900, 0, 0, 0, 0, 25, 0, 0, 0, 0, 8192, 68, 184, 8188, 3, 8188, 0, 0, 7789, 8189, 8078, 0, 0, 0, 0, 0, 35, 8191, 0, 5, 0, 0, 5, 0, 0, 0, 0, 8155, 46, 7464, 0, 3048, 0, 7931, 4112, 8032] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16170, 16382, 16384, 16384] dup_rows_per_dst=[16720, 8868, 11854, 16880, 15570, 16198, 16314, 28884] combine_rows_per_src=[16384, 16384, 16384, 16384, 16170, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[1, 2, 2, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123658 payload_rows_remote=107274 dup_rows=138486 dup_rows_remote=111474 dedup_ratio=0.528282 router_tokens_per_expert=[0, 8, 0, 8181, 12, 0, 0, 185, 6, 5501, 0, 7, 8184, 0, 2, 0, 0, 8112, 7414, 3, 8, 0, 0, 27, 7, 0, 1, 0, 8174, 3, 51, 8192, 0, 8109, 19, 0, 7412, 8185, 8185, 0, 0, 0, 0, 0, 0, 8180, 0, 11, 7, 257, 29, 0, 0, 1, 1, 8162, 0, 7441, 0, 3962, 18, 8186, 687, 8142] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 8974] dup_rows_per_dst=[16764, 27012, 29914, 16472, 14852, 16212, 16900, 360] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 8974] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937050112 dispatch_meta_bytes=818192 combine_payload_bytes=937050112 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=937868304 ingress_bytes=1729773568 link_bytes=1729773568 token_bytes=1874918416 weight_movement_bytes=792723456 total_nvlink_bytes=2667641872 weight_to_token_ratio=0.422804 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=24576 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.686009938e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=911654912 dispatch_meta_bytes=917712 combine_payload_bytes=911654912 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=912572624 ingress_bytes=1175896064 link_bytes=1175896064 token_bytes=1824227536 weight_movement_bytes=264241152 total_nvlink_bytes=2088468688 weight_to_token_ratio=0.144851 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.584527115e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=113248 dedup_ratio=0.500092 router_tokens_per_expert=[0, 0, 3, 8192, 0, 4, 26, 4, 0, 3986, 0, 0, 8191, 0, 0, 0, 0, 8192, 4807, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 77, 7, 8192, 5, 8182, 0, 3, 8189, 8192, 8121, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 1016, 0, 0, 10, 0, 7, 8192, 0, 3621, 1, 7548, 0, 8177, 3732, 8120] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16376, 16384, 16368] dup_rows_per_dst=[16416, 23218, 17848, 16558, 16252, 16160, 18450, 6194] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16376, 16384, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937771008 dispatch_meta_bytes=918256 combine_payload_bytes=937771008 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938689264 ingress_bytes=1290092544 link_bytes=1290092544 token_bytes=1876460272 weight_movement_bytes=352321536 total_nvlink_bytes=2228781808 weight_to_token_ratio=0.187759 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223979735e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114724 dedup_ratio=0.500046 router_tokens_per_expert=[0, 8022, 17, 0, 8160, 0, 0, 25, 8159, 8192, 8179, 40, 8179, 13, 403, 0, 52, 7, 0, 9, 209, 0, 3915, 6, 0, 17, 2223, 0, 0, 24, 8050, 20, 0, 15, 8185, 0, 0, 1, 3, 0, 8185, 0, 12, 0, 2232, 0, 0, 8147, 0, 7673, 0, 0, 7, 0, 8175, 28, 7973, 196, 2, 0, 0, 0, 8168, 8149] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16378, 16382, 16382] dup_rows_per_dst=[16066, 10900, 22890, 20644, 16360, 12582, 15384, 16258] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16378, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114494 dedup_ratio=0.500023 router_tokens_per_expert=[0, 0, 2, 8192, 0, 12, 0, 227, 0, 8113, 0, 0, 8192, 0, 13, 0, 0, 8191, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 28, 3, 8192, 11, 8192, 0, 0, 8180, 8188, 8139, 0, 0, 0, 0, 0, 2, 8167, 0, 0, 0, 99, 0, 1, 0, 5, 0, 8191, 25, 7935, 0, 6863, 0, 8017, 1226, 8172] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16770, 16252, 604, 16446, 16280, 16306, 16584, 31836] combine_rows_per_src=[16384, 16384, 16382, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127784 payload_rows_remote=111404 dup_rows=134360 dup_rows_remote=118290 dedup_ratio=0.512543 router_tokens_per_expert=[8, 0, 8, 8139, 18, 0, 43, 26, 9, 3676, 51, 0, 8091, 0, 0, 82, 79, 7815, 7381, 0, 49, 37, 13, 306, 0, 1, 0, 0, 8095, 455, 810, 8166, 40, 8173, 394, 18, 8028, 8132, 7954, 26, 0, 18, 8, 0, 87, 8159, 0, 60, 2, 2, 38, 74, 0, 120, 21, 7484, 103, 5884, 39, 7816, 13, 6935, 1445, 6641] payload_rows_per_dst=[16380, 16318, 16384, 16366, 16384, 16384, 15340, 14228] dup_rows_per_dst=[16070, 18424, 27458, 19790, 23194, 18044, 8646, 2734] combine_rows_per_src=[16380, 16318, 16384, 16366, 16384, 16384, 15340, 14228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=878788608 dispatch_meta_bytes=874992 combine_payload_bytes=878788608 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=879663600 ingress_bytes=1054949376 link_bytes=1054949376 token_bytes=1758452208 weight_movement_bytes=176160768 total_nvlink_bytes=1934612976 weight_to_token_ratio=0.100179 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.907276148e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=911648 combine_payload_bytes=939327488 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940239136 ingress_bytes=1115488256 link_bytes=1115488256 token_bytes=1879566624 weight_movement_bytes=176160768 total_nvlink_bytes=2055727392 weight_to_token_ratio=0.093724 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246268606e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917600 combine_payload_bytes=939425792 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940343392 ingress_bytes=1203666944 link_bytes=1203666944 token_bytes=1879769184 weight_movement_bytes=264241152 total_nvlink_bytes=2144010336 weight_to_token_ratio=0.140571 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740032451e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=916704 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940391648 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879866592 weight_movement_bytes=176160768 total_nvlink_bytes=2056027360 weight_to_token_ratio=0.093709 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=912621568 dispatch_meta_bytes=918776 combine_payload_bytes=912621568 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=913540344 ingress_bytes=1088782336 link_bytes=1088782336 token_bytes=1826161912 weight_movement_bytes=176160768 total_nvlink_bytes=2002322680 weight_to_token_ratio=0.096465 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.096726601e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127700 payload_rows_remote=111402 dup_rows=134444 dup_rows_remote=108962 dedup_ratio=0.512863 router_tokens_per_expert=[15, 0, 3, 7546, 2136, 81, 15, 267, 8164, 0, 1, 27, 0, 0, 4956, 8176, 0, 30, 13, 0, 13, 8116, 8, 8166, 20, 5433, 0, 4973, 224, 116, 0, 9, 2, 8186, 1, 6586, 0, 8134, 524, 170, 0, 5451, 1, 2057, 8192, 0, 8, 5, 0, 6265, 64, 3, 3, 8032, 57, 2932, 9, 0, 3, 7496, 0, 5, 325, 8053] payload_rows_per_dst=[15048, 14842, 16384, 15982, 16378, 16384, 16384, 16298] dup_rows_per_dst=[3166, 6514, 16308, 5568, 16942, 28950, 31514, 25482] combine_rows_per_src=[15048, 14842, 16384, 15982, 16378, 16384, 16384, 16298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114678 dup_rows=131094 dup_rows_remote=109916 dedup_ratio=0.500084 router_tokens_per_expert=[7, 3575, 0, 8182, 7644, 72, 5, 4237, 8188, 4, 0, 0, 5, 0, 779, 8068, 0, 0, 16, 0, 1, 8188, 3, 4330, 3, 1, 77, 8137, 0, 2322, 68, 0, 8, 8189, 12, 8191, 0, 8167, 8, 0, 3, 7594, 121, 0, 8044, 3255, 2, 0, 0, 1850, 0, 0, 0, 8185, 5, 0, 25, 3, 2, 5281, 39, 0, 0, 8176] payload_rows_per_dst=[16380, 16384, 16382, 16372, 16384, 16384, 16380, 16384] dup_rows_per_dst=[31064, 17704, 8694, 21178, 16432, 21654, 3700, 10668] combine_rows_per_src=[16380, 16384, 16382, 16372, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129806 payload_rows_remote=113422 dup_rows=132338 dup_rows_remote=118050 dedup_ratio=0.504829 router_tokens_per_expert=[0, 8, 0, 7955, 5793, 1, 0, 3856, 8189, 2, 0, 1, 0, 0, 3462, 8191, 0, 4, 16, 0, 0, 8189, 0, 8130, 23, 198, 127, 127, 14, 5738, 683, 0, 0, 8189, 0, 5041, 18, 8185, 749, 72, 0, 7078, 0, 26, 8192, 0, 40, 0, 0, 8032, 0, 0, 0, 8092, 16, 358, 0, 0, 0, 8191, 0, 0, 0, 8086] payload_rows_per_dst=[16006, 16384, 16382, 15498, 16384, 16384, 16384, 16384] dup_rows_per_dst=[19220, 20568, 16296, 5554, 18962, 14288, 21280, 16170] combine_rows_per_src=[16006, 16384, 16382, 15498, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=912605184 dispatch_meta_bytes=881456 combine_payload_bytes=912605184 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=913486640 ingress_bytes=1088765952 link_bytes=1088765952 token_bytes=1826091824 weight_movement_bytes=176160768 total_nvlink_bytes=2002252592 weight_to_token_ratio=0.096469 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.096634858e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130792 payload_rows_remote=114408 dup_rows=131352 dup_rows_remote=107088 dedup_ratio=0.501068 router_tokens_per_expert=[2, 3562, 0, 8064, 4044, 273, 4, 2189, 8183, 0, 0, 16, 0, 0, 6829, 8081, 0, 4, 82, 2, 0, 8030, 0, 8192, 11, 1970, 55, 5822, 49, 1179, 212, 0, 1, 8089, 281, 7927, 0, 8189, 0, 6, 3, 2179, 0, 2, 8165, 780, 8, 0, 1, 4092, 0, 1, 0, 8192, 1, 9, 1, 6, 0, 8155, 6, 0, 6, 8117] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16140, 16348, 16384, 16384] dup_rows_per_dst=[25462, 24264, 20518, 18590, 12186, 5926, 8208, 16198] combine_rows_per_src=[16384, 16384, 16384, 16384, 16140, 16348, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=898376 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940340552 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879782728 weight_movement_bytes=176160768 total_nvlink_bytes=2055943496 weight_to_token_ratio=0.093713 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.023674 gemm_active_groups=19 gemm_peak_group_rows=27136 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130850 payload_rows_remote=114492 dup_rows=131294 dup_rows_remote=114960 dedup_ratio=0.500847 router_tokens_per_expert=[0, 47, 0, 7800, 8082, 32, 0, 3104, 8180, 0, 0, 0, 0, 0, 2400, 8143, 0, 1, 8, 0, 0, 8167, 0, 8170, 0, 2, 0, 8072, 3, 3068, 217, 0, 0, 8192, 0, 8054, 0, 8192, 105, 13, 0, 5789, 0, 202, 8192, 165, 9, 0, 0, 2468, 0, 0, 0, 8179, 0, 49, 0, 0, 0, 7855, 0, 0, 25, 8087] payload_rows_per_dst=[16298, 16384, 16358, 16384, 16384, 16384, 16384, 16274] dup_rows_per_dst=[21832, 21062, 16334, 22724, 16344, 12330, 5008, 15660] combine_rows_per_src=[16298, 16384, 16358, 16384, 16384, 16384, 16384, 16274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130944 payload_rows_remote=114560 dup_rows=131200 dup_rows_remote=114872 dedup_ratio=0.500488 router_tokens_per_expert=[14, 15, 0, 8177, 10, 0, 39, 28, 0, 7443, 0, 0, 8172, 14, 33, 0, 1, 7854, 5258, 0, 0, 0, 0, 0, 1, 11, 2, 0, 8184, 0, 5, 8174, 0, 8169, 0, 0, 8181, 8170, 8152, 23, 8, 0, 19, 0, 0, 8192, 8, 25, 0, 78, 1, 0, 0, 0, 0, 8192, 0, 7536, 0, 4497, 2, 7984, 1382, 7018] payload_rows_per_dst=[16384, 16384, 16272, 16384, 16384, 16384, 16384, 16368] dup_rows_per_dst=[16564, 14940, 9954, 16370, 16328, 16458, 16498, 24088] combine_rows_per_src=[16384, 16384, 16272, 16384, 16384, 16384, 16384, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131016 payload_rows_remote=114632 dup_rows=131128 dup_rows_remote=105252 dedup_ratio=0.500214 router_tokens_per_expert=[0, 0, 0, 8183, 1885, 0, 0, 944, 8192, 0, 0, 0, 0, 1, 5980, 8192, 0, 0, 1, 0, 0, 8179, 0, 7500, 0, 5486, 86, 7767, 0, 1719, 56, 0, 0, 8192, 0, 8026, 0, 8190, 172, 0, 0, 5116, 0, 0, 8192, 72, 3, 0, 0, 4758, 0, 1, 0, 8178, 0, 1, 0, 0, 0, 7804, 0, 0, 6, 8190] payload_rows_per_dst=[16366, 16384, 16384, 16346, 16384, 16384, 16384, 16384] dup_rows_per_dst=[5658, 12020, 14976, 13882, 24226, 18516, 25876, 15974] combine_rows_per_src=[16366, 16384, 16384, 16346, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 2, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29088519520 prefetch_bytes=3170893824 reduce_bytes=2730491904 weight_movement_bytes=5901385728 egress_bytes=14551324000 ingress_bytes=20438581248 link_bytes=20438581248 total_nvlink_bytes=34989905248 weight_to_token_ratio=0.202877 prefetch_share_of_ingress=0.155143 dedup_ratio=0.514602 prefetch_slots=72 unused_prefetch_slots=440 slot_utilization=0.140625 reduce_slots=31 remote_demand=36 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.019886 gemm_inner_pad_upper_bound=39370 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.152970219e-05 prefetch_bytes_per_gemm_flop=1.109730114e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=929153024 dispatch_meta_bytes=925888 combine_payload_bytes=929153024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=930078912 ingress_bytes=1105313792 link_bytes=1105313792 token_bytes=1859231936 weight_movement_bytes=176160768 total_nvlink_bytes=2035392704 weight_to_token_ratio=0.094749 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.189295855e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130292 payload_rows_remote=113910 dup_rows=131852 dup_rows_remote=113582 dedup_ratio=0.502975 router_tokens_per_expert=[5, 0, 11, 7952, 2821, 0, 44, 2290, 8122, 0, 18, 28, 8, 0, 825, 8067, 70, 48, 0, 26, 50, 8117, 59, 4644, 64, 986, 0, 8090, 522, 2985, 0, 0, 0, 8177, 187, 6540, 0, 8105, 1970, 488, 2, 5451, 0, 891, 8131, 0, 89, 0, 34, 8149, 19, 0, 7, 7304, 86, 3885, 0, 26, 45, 7633, 13, 36, 95, 7857] payload_rows_per_dst=[16382, 16296, 16384, 16382, 16064, 16330, 16384, 16070] dup_rows_per_dst=[18270, 9738, 23484, 25244, 4394, 12798, 22584, 15340] combine_rows_per_src=[16382, 16296, 16384, 16382, 16064, 16330, 16384, 16070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937230336 dispatch_meta_bytes=885984 combine_payload_bytes=937230336 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=938116320 ingress_bytes=1465712640 link_bytes=1465712640 token_bytes=1875346656 weight_movement_bytes=528482304 total_nvlink_bytes=2403828960 weight_to_token_ratio=0.281805 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.207378965e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29884987328 prefetch_bytes=2730491904 reduce_bytes=2025848832 weight_movement_bytes=4756340736 egress_bytes=14949840832 ingress_bytes=19691487232 link_bytes=19691487232 total_nvlink_bytes=34641328064 weight_to_token_ratio=0.159155 prefetch_share_of_ingress=0.138664 dedup_ratio=0.503147 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=23 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.019946 gemm_inner_pad_upper_bound=38227 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.891506804e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937918464 dispatch_meta_bytes=917808 combine_payload_bytes=937918464 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938836272 ingress_bytes=1114079232 link_bytes=1114079232 token_bytes=1876754736 weight_movement_bytes=176160768 total_nvlink_bytes=2052915504 weight_to_token_ratio=0.093865 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.238378660e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938475520 dispatch_meta_bytes=917728 combine_payload_bytes=938475520 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=939393248 ingress_bytes=1643118592 link_bytes=1643118592 token_bytes=1877868768 weight_movement_bytes=704643072 total_nvlink_bytes=2582511840 weight_to_token_ratio=0.375236 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.200778244e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939065344 dispatch_meta_bytes=879536 combine_payload_bytes=939065344 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939944880 ingress_bytes=1115226112 link_bytes=1115226112 token_bytes=1879010224 weight_movement_bytes=176160768 total_nvlink_bytes=2055170992 weight_to_token_ratio=0.093752 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244800709e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=933150720 dispatch_meta_bytes=909968 combine_payload_bytes=933150720 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=934060688 ingress_bytes=1197391872 link_bytes=1197391872 token_bytes=1867211408 weight_movement_bytes=264241152 total_nvlink_bytes=2131452560 weight_to_token_ratio=0.141516 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.704894667e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29617279336 prefetch_bytes=2730491904 reduce_bytes=4404019200 weight_movement_bytes=7134511104 egress_bytes=14815809896 ingress_bytes=21935980544 link_bytes=21935980544 total_nvlink_bytes=36751790440 weight_to_token_ratio=0.240890 prefetch_share_of_ingress=0.124475 dedup_ratio=0.507011 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=50 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020360 gemm_inner_pad_upper_bound=37592 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.677020907e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29885839912 prefetch_bytes=3258974208 reduce_bytes=2113929216 weight_movement_bytes=5372903424 egress_bytes=14950251048 ingress_bytes=20308492288 link_bytes=20308492288 total_nvlink_bytes=35258743336 weight_to_token_ratio=0.179781 prefetch_share_of_ingress=0.160473 dedup_ratio=0.503621 prefetch_slots=74 unused_prefetch_slots=438 slot_utilization=0.144531 reduce_slots=24 remote_demand=37 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.019709 gemm_inner_pad_upper_bound=39370 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.107442477e-05 prefetch_bytes_per_gemm_flop=1.140555950e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29819011952 prefetch_bytes=2642411520 reduce_bytes=2642411520 weight_movement_bytes=5284823040 egress_bytes=14916862832 ingress_bytes=20186972160 link_bytes=20186972160 total_nvlink_bytes=35103834992 weight_to_token_ratio=0.177230 prefetch_share_of_ingress=0.130897 dedup_ratio=0.504052 prefetch_slots=60 unused_prefetch_slots=452 slot_utilization=0.117188 reduce_slots=30 remote_demand=30 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.019827 gemm_inner_pad_upper_bound=38735 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.064913602e-05 prefetch_bytes_per_gemm_flop=9.247750947e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29846309640 prefetch_bytes=2642411520 reduce_bytes=2818572288 weight_movement_bytes=5460983808 egress_bytes=14930365192 ingress_bytes=20376928256 link_bytes=20376928256 total_nvlink_bytes=35307293448 weight_to_token_ratio=0.182970 prefetch_share_of_ingress=0.129677 dedup_ratio=0.503520 prefetch_slots=60 unused_prefetch_slots=452 slot_utilization=0.117188 reduce_slots=32 remote_demand=30 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.019709 gemm_inner_pad_upper_bound=38862 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.131393280e-05 prefetch_bytes_per_gemm_flop=9.247750947e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29295850992 prefetch_bytes=2994733056 reduce_bytes=2378170368 weight_movement_bytes=5372903424 egress_bytes=14655092208 ingress_bytes=20013662208 link_bytes=20013662208 total_nvlink_bytes=34668754416 weight_to_token_ratio=0.183402 prefetch_share_of_ingress=0.149634 dedup_ratio=0.511509 prefetch_slots=68 unused_prefetch_slots=444 slot_utilization=0.132812 reduce_slots=27 remote_demand=34 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.019650 gemm_inner_pad_upper_bound=39497 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.004259641e-05 prefetch_bytes_per_gemm_flop=1.048078441e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114644 dup_rows=131116 dup_rows_remote=108960 dedup_ratio=0.500168 router_tokens_per_expert=[24, 68, 0, 8124, 663, 5, 0, 3420, 8181, 0, 47, 15, 0, 17, 2299, 8049, 0, 0, 0, 0, 0, 8175, 1, 7262, 0, 3888, 104, 8154, 0, 409, 84, 37, 0, 8165, 0, 8187, 0, 8183, 87, 0, 18, 8168, 9, 1, 8151, 84, 111, 0, 10, 6427, 1, 13, 0, 8191, 0, 3, 0, 8, 2, 8021, 1, 14, 14, 8177] payload_rows_per_dst=[16384, 16384, 16352, 16384, 16384, 16384, 16384, 16372] dup_rows_per_dst=[8224, 20832, 14524, 19672, 22156, 16700, 12906, 16102] combine_rows_per_src=[16384, 16384, 16352, 16384, 16384, 16384, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 2, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=894416 combine_payload_bytes=939163648 prefetch_bytes=88080384 reduce_bytes=704643072 egress_bytes=940058064 ingress_bytes=1731887104 link_bytes=1731887104 token_bytes=1879221712 weight_movement_bytes=792723456 total_nvlink_bytes=2671945168 weight_to_token_ratio=0.421836 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=28032 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.697844857e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29908929304 prefetch_bytes=2818572288 reduce_bytes=3875536896 weight_movement_bytes=6694109184 egress_bytes=14961822488 ingress_bytes=21641216000 link_bytes=21641216000 total_nvlink_bytes=36603038488 weight_to_token_ratio=0.223816 prefetch_share_of_ingress=0.130241 dedup_ratio=0.502647 prefetch_slots=64 unused_prefetch_slots=448 slot_utilization=0.125000 reduce_slots=44 remote_demand=32 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.019590 gemm_inner_pad_upper_bound=39116 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.573861007e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 anomalies=none - [2026-09-16 21:39:30] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 615.7 | throughput per GPU (TFLOP/s/GPU): 541.5 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.316022E+01 | loss scale: 1.0 | grad norm: 38.065 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125782 payload_rows_remote=110124 dup_rows=136362 dup_rows_remote=120824 dedup_ratio=0.520180 router_tokens_per_expert=[28, 352, 5281, 18, 1593, 6, 2490, 22, 5385, 104, 566, 1944, 1785, 353, 2, 2274, 448, 5143, 4345, 1988, 7523, 1713, 620, 3445, 2, 7234, 196, 1801, 615, 40, 257, 2868, 7545, 246, 69, 1836, 2629, 36, 1279, 4683, 4457, 4965, 837, 477, 13, 3225, 2831, 3758, 162, 2938, 1001, 14, 17, 169, 5863, 413, 5, 7312, 7342, 3134, 354, 2401, 605, 15] payload_rows_per_dst=[16272, 14702, 15792, 15740, 15632, 15658, 15652, 16334] dup_rows_per_dst=[18354, 10124, 19196, 10286, 21430, 15538, 15432, 26002] combine_rows_per_src=[16272, 14702, 15792, 15740, 15632, 15658, 15652, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124894 payload_rows_remote=109054 dup_rows=137250 dup_rows_remote=116960 dedup_ratio=0.523567 router_tokens_per_expert=[28, 179, 4935, 19, 1194, 1, 2214, 14, 5967, 173, 952, 2411, 1430, 524, 16, 1807, 524, 4297, 4660, 2403, 7718, 1867, 623, 3691, 1, 7253, 128, 1225, 1157, 98, 470, 2674, 7515, 216, 126, 2172, 2941, 45, 1332, 4471, 5120, 5085, 1153, 474, 9, 3332, 2141, 3101, 258, 2836, 992, 54, 74, 159, 5903, 488, 10, 6770, 6708, 2362, 586, 2475, 1494, 17] payload_rows_per_dst=[16276, 16096, 15840, 16368, 13486, 15538, 15710, 15580] dup_rows_per_dst=[16328, 23880, 20290, 24674, 9120, 15122, 15988, 11848] combine_rows_per_src=[16276, 16096, 15840, 16368, 13486, 15538, 15710, 15580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125790 payload_rows_remote=109598 dup_rows=136354 dup_rows_remote=117162 dedup_ratio=0.520149 router_tokens_per_expert=[32, 226, 5042, 12, 959, 3, 2145, 3, 5702, 237, 771, 1923, 1607, 495, 2, 1711, 593, 4499, 4975, 1850, 7696, 2084, 511, 3349, 0, 7496, 171, 1287, 638, 52, 323, 2763, 7509, 190, 97, 2222, 2848, 39, 1167, 4229, 4960, 5297, 1261, 650, 4, 3376, 2675, 4421, 155, 2774, 1225, 12, 57, 83, 6160, 483, 6, 6713, 7387, 2518, 370, 2014, 982, 31] payload_rows_per_dst=[16260, 15992, 15860, 16192, 14210, 15912, 15954, 15410] dup_rows_per_dst=[15976, 21760, 19862, 19192, 12468, 18782, 16538, 11776] combine_rows_per_src=[16260, 15992, 15860, 16192, 14210, 15912, 15954, 15410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126008 payload_rows_remote=110138 dup_rows=136136 dup_rows_remote=119854 dedup_ratio=0.519318 router_tokens_per_expert=[17, 242, 4409, 11, 991, 0, 2411, 8, 6102, 109, 731, 2646, 1383, 567, 2, 1949, 547, 4770, 4583, 1634, 7788, 2010, 678, 3235, 1, 7722, 73, 1144, 523, 23, 221, 3020, 7474, 158, 88, 2631, 3052, 25, 771, 4490, 5340, 4961, 509, 379, 10, 3458, 2928, 4030, 137, 3444, 1334, 31, 16, 50, 5788, 315, 1, 7379, 7230, 2264, 448, 2050, 702, 29] payload_rows_per_dst=[16310, 15038, 14292, 15986, 16344, 15840, 15870, 16328] dup_rows_per_dst=[15444, 11940, 11292, 9468, 30364, 17468, 16282, 23878] combine_rows_per_src=[16310, 15038, 14292, 15986, 16344, 15840, 15870, 16328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124162 payload_rows_remote=107872 dup_rows=137982 dup_rows_remote=112830 dedup_ratio=0.526360 router_tokens_per_expert=[27, 383, 5139, 5, 1095, 1, 1727, 25, 5493, 193, 565, 2342, 2050, 190, 7, 1561, 572, 4084, 5141, 1843, 7479, 1993, 786, 3650, 3, 6725, 299, 1567, 689, 79, 397, 3303, 7371, 199, 163, 2110, 3130, 38, 1208, 4365, 5098, 5856, 1294, 705, 33, 2188, 2550, 4171, 345, 3540, 1028, 48, 9, 100, 5457, 507, 1, 6916, 6940, 2321, 583, 2056, 1222, 107] payload_rows_per_dst=[16188, 14736, 13844, 15364, 16226, 15440, 16074, 16290] dup_rows_per_dst=[15574, 10066, 10388, 10760, 28790, 16402, 20850, 25152] combine_rows_per_src=[16188, 14736, 13844, 15364, 16226, 15440, 16074, 16290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124990 payload_rows_remote=108914 dup_rows=137154 dup_rows_remote=114280 dedup_ratio=0.523201 router_tokens_per_expert=[40, 188, 3809, 118, 1497, 0, 1766, 28, 5325, 417, 834, 1641, 1437, 500, 17, 1988, 573, 3704, 4433, 1992, 7612, 2591, 730, 3590, 15, 6990, 159, 2256, 614, 41, 342, 2700, 7302, 161, 161, 2071, 2797, 38, 1153, 3913, 4719, 4981, 1015, 385, 81, 3448, 3090, 4920, 177, 2693, 1719, 77, 61, 206, 5993, 426, 3, 7165, 7316, 2973, 589, 2373, 1007, 112] payload_rows_per_dst=[16174, 16076, 15792, 16358, 13164, 15940, 15730, 15756] dup_rows_per_dst=[13942, 22874, 19434, 24480, 7424, 19376, 16936, 12688] combine_rows_per_src=[16174, 16076, 15792, 16358, 13164, 15940, 15730, 15756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[4, 0, 0, 0, 2, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=902135808 dispatch_meta_bytes=923792 combine_payload_bytes=902135808 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=903059600 ingress_bytes=1430618112 link_bytes=1430618112 token_bytes=1805195408 weight_movement_bytes=528482304 total_nvlink_bytes=2333677712 weight_to_token_ratio=0.292756 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=18944 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.010864258e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=893370368 dispatch_meta_bytes=904056 combine_payload_bytes=893370368 prefetch_bytes=88080384 reduce_bytes=704643072 egress_bytes=894274424 ingress_bytes=1686093824 link_bytes=1686093824 token_bytes=1787644792 weight_movement_bytes=792723456 total_nvlink_bytes=2580368248 weight_to_token_ratio=0.443446 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.023674 gemm_active_groups=17 gemm_peak_group_rows=16768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.441421604e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=897826816 dispatch_meta_bytes=907040 combine_payload_bytes=897826816 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=898733856 ingress_bytes=1073987584 link_bytes=1073987584 token_bytes=1796560672 weight_movement_bytes=176160768 total_nvlink_bytes=1972721440 weight_to_token_ratio=0.098054 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=29312 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.013882166e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=902250496 dispatch_meta_bytes=919968 combine_payload_bytes=902250496 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=903170464 ingress_bytes=1254572032 link_bytes=1254572032 token_bytes=1805420960 weight_movement_bytes=352321536 total_nvlink_bytes=2157742496 weight_to_token_ratio=0.195146 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.019886 gemm_active_groups=22 gemm_peak_group_rows=24832 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.025079695e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122802 payload_rows_remote=106546 dup_rows=139342 dup_rows_remote=118640 dedup_ratio=0.531548 router_tokens_per_expert=[30, 157, 4859, 11, 925, 0, 1819, 8, 5603, 261, 688, 2046, 1631, 591, 5, 2177, 512, 4758, 4985, 2317, 7389, 2072, 693, 3555, 2, 7022, 231, 1580, 732, 54, 259, 3093, 7576, 166, 127, 2704, 3552, 40, 962, 4089, 5154, 4820, 571, 322, 11, 2778, 3366, 3284, 207, 2664, 1382, 25, 31, 78, 6449, 620, 3, 7122, 7156, 2032, 506, 2212, 960, 38] payload_rows_per_dst=[16256, 16020, 15302, 16354, 13802, 15598, 15798, 13672] dup_rows_per_dst=[20702, 24296, 17122, 24744, 9478, 17936, 16754, 8310] combine_rows_per_src=[16256, 16020, 15302, 16354, 13802, 15598, 15798, 13672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=883687424 dispatch_meta_bytes=882808 combine_payload_bytes=883687424 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=884570232 ingress_bytes=1324089344 link_bytes=1324089344 token_bytes=1768257656 weight_movement_bytes=440401920 total_nvlink_bytes=2208659576 weight_to_token_ratio=0.249060 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=28032 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.414347624e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=892223488 dispatch_meta_bytes=892776 combine_payload_bytes=892223488 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=893116264 ingress_bytes=1068384256 link_bytes=1068384256 token_bytes=1785339752 weight_movement_bytes=176160768 total_nvlink_bytes=1961500520 weight_to_token_ratio=0.098671 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=23168 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.982505868e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=872824832 dispatch_meta_bytes=900744 combine_payload_bytes=872824832 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=873725576 ingress_bytes=1225146368 link_bytes=1225146368 token_bytes=1746550408 weight_movement_bytes=352321536 total_nvlink_bytes=2098871944 weight_to_token_ratio=0.201724 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.017992 gemm_active_groups=22 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.860308260e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114656 dup_rows=131110 dup_rows_remote=114500 dedup_ratio=0.500145 router_tokens_per_expert=[1, 4, 0, 9, 0, 1, 29, 8187, 10, 10, 8133, 30, 8189, 8165, 8188, 1, 0, 4, 1, 8180, 127, 8182, 0, 0, 7179, 32, 19, 3, 1788, 0, 26, 5, 8122, 8123, 0, 20, 0, 0, 2, 8129, 0, 0, 4, 8187, 8168, 0, 3, 14, 8, 8119, 2, 11, 2, 5, 37, 1, 24, 2, 0, 7400, 3, 8176, 7, 0] payload_rows_per_dst=[16384, 16380, 16378, 16382, 16366, 16376, 16384, 16384] dup_rows_per_dst=[16456, 16326, 16610, 17962, 16182, 16376, 16358, 14840] combine_rows_per_src=[16384, 16380, 16378, 16382, 16366, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131014 payload_rows_remote=114630 dup_rows=131130 dup_rows_remote=114582 dedup_ratio=0.500221 router_tokens_per_expert=[2, 7, 12, 2, 1, 11, 2, 8181, 11, 20, 7517, 8, 8108, 8142, 8166, 8, 21, 0, 2, 8135, 233, 8178, 2, 11, 8149, 103, 33, 5, 1738, 3, 12, 2, 8100, 8153, 13, 1, 0, 2, 2, 8181, 12, 0, 7, 8178, 8176, 2, 18, 73, 0, 8138, 43, 45, 1, 5, 51, 0, 14, 0, 4, 6865, 0, 8161, 2, 0] payload_rows_per_dst=[16380, 16376, 16382, 16378, 16380, 16384, 16384, 16350] dup_rows_per_dst=[16362, 14998, 16782, 19956, 16324, 16548, 16418, 13742] combine_rows_per_src=[16380, 16376, 16382, 16378, 16380, 16384, 16384, 16350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114224 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 1, 2, 6, 0, 0, 8192, 7, 5, 8159, 1, 8183, 8167, 8189, 5, 5, 1, 3, 8190, 21, 8177, 0, 6, 8109, 34, 0, 2, 293, 0, 12, 0, 8190, 8127, 23, 1, 14, 8, 4, 8038, 1, 4, 1, 8192, 8190, 0, 2, 0, 0, 8186, 24, 0, 0, 4, 0, 2, 0, 1, 1, 8093, 0, 8188, 0, 8] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16376, 16384, 16384, 16380] dup_rows_per_dst=[16384, 16344, 16422, 16860, 16054, 16396, 16422, 16202] combine_rows_per_src=[16384, 16384, 16384, 16384, 16376, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126150 payload_rows_remote=110510 dup_rows=135994 dup_rows_remote=111428 dedup_ratio=0.518776 router_tokens_per_expert=[16, 93, 4711, 4, 1662, 1, 2132, 5, 4915, 156, 451, 2049, 1275, 420, 1, 1422, 398, 5332, 4615, 2245, 7420, 2116, 491, 3722, 1, 7250, 144, 1383, 636, 29, 274, 3372, 7634, 232, 66, 2134, 3496, 41, 1243, 5257, 4388, 4749, 1192, 381, 7, 2761, 3012, 4056, 192, 3550, 1136, 26, 5, 86, 5941, 603, 6, 7119, 7067, 1899, 327, 2329, 1365, 31] payload_rows_per_dst=[16182, 15128, 15952, 15738, 15640, 15726, 15914, 15870] dup_rows_per_dst=[15906, 13416, 21886, 10440, 24566, 15868, 16662, 17250] combine_rows_per_src=[16182, 15128, 15952, 15738, 15640, 15726, 15914, 15870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114682 dup_rows=131080 dup_rows_remote=114698 dedup_ratio=0.500031 router_tokens_per_expert=[1, 1, 0, 0, 0, 0, 0, 8192, 51, 46, 8189, 0, 8164, 8189, 8169, 0, 1, 0, 1, 8191, 167, 8187, 0, 5, 8073, 7, 0, 0, 438, 32, 0, 1, 8188, 8035, 5, 0, 0, 2, 0, 8160, 4, 5, 0, 8176, 8188, 0, 0, 2, 0, 8191, 0, 2, 0, 0, 0, 0, 1, 3, 0, 7815, 0, 8189, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16382, 16382] dup_rows_per_dst=[16346, 16516, 16720, 17090, 16020, 16370, 16382, 15636] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114656 dup_rows=131104 dup_rows_remote=114620 dedup_ratio=0.500122 router_tokens_per_expert=[149, 0, 1, 0, 54, 66, 29, 8160, 76, 37, 8181, 0, 8158, 8164, 8189, 4, 34, 0, 20, 8188, 227, 8163, 2, 3, 7174, 3, 0, 1, 1623, 12, 25, 4, 8145, 7524, 7, 1, 2, 1, 67, 8138, 2, 4, 0, 8177, 8118, 1, 26, 0, 0, 8131, 1, 60, 1, 9, 8, 0, 3, 0, 0, 7711, 1, 8187, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16374, 16378, 16374, 16384, 16378] dup_rows_per_dst=[16904, 16484, 16890, 17654, 15102, 16282, 16362, 15426] combine_rows_per_src=[16384, 16384, 16384, 16374, 16378, 16374, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=106122 dedup_ratio=0.500023 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 8189, 116, 11, 8182, 3, 8185, 8190, 8192, 2, 1, 0, 1, 8190, 633, 8177, 0, 0, 7280, 4, 3, 0, 749, 3, 0, 1, 8190, 8188, 4, 0, 2, 1, 3, 8180, 0, 0, 0, 8184, 8190, 0, 1, 0, 0, 8188, 0, 3, 0, 1, 4, 0, 0, 0, 0, 7631, 2, 8187, 0, 0] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[15826, 11320, 15970, 11060, 18904, 16476, 16566, 24956] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=916624 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940178576 ingress_bytes=1379663872 link_bytes=1379663872 token_bytes=1879440528 weight_movement_bytes=440401920 total_nvlink_bytes=2319842448 weight_to_token_ratio=0.234326 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725541783e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[3, 0, 1, 1, 0, 1, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=915600 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940341392 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879767184 weight_movement_bytes=176160768 total_nvlink_bytes=2055927952 weight_to_token_ratio=0.093714 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939048960 dispatch_meta_bytes=916848 combine_payload_bytes=939048960 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=939965808 ingress_bytes=1115209728 link_bytes=1115209728 token_bytes=1879014768 weight_movement_bytes=176160768 total_nvlink_bytes=2055175536 weight_to_token_ratio=0.093752 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244708965e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=917520 combine_payload_bytes=939474944 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940392464 ingress_bytes=1203716096 link_bytes=1203716096 token_bytes=1879867408 weight_movement_bytes=264241152 total_nvlink_bytes=2144108560 weight_to_token_ratio=0.140564 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740307682e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=917104 combine_payload_bytes=939261952 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=940179056 ingress_bytes=1820065792 link_bytes=1820065792 token_bytes=1879441008 weight_movement_bytes=880803840 total_nvlink_bytes=2760244848 weight_to_token_ratio=0.468652 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019160870e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114660 dedup_ratio=0.500084 router_tokens_per_expert=[3, 0, 2, 0, 6, 0, 3, 8185, 48, 1, 8066, 0, 8154, 8188, 8189, 13, 0, 2, 0, 8190, 39, 8191, 4, 4, 7177, 36, 17, 0, 2253, 0, 0, 0, 8179, 8149, 1, 4, 0, 0, 0, 8067, 3, 3, 2, 8188, 8190, 0, 0, 36, 0, 8187, 0, 0, 1, 0, 2, 0, 1, 4, 0, 7102, 0, 8182, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16382, 16384, 16380, 16372] dup_rows_per_dst=[16434, 22780, 15314, 13680, 16244, 16262, 16174, 14206] combine_rows_per_src=[16384, 16380, 16384, 16384, 16382, 16384, 16380, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=883216 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940358160 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879833104 weight_movement_bytes=176160768 total_nvlink_bytes=2055993872 weight_to_token_ratio=0.093711 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=905297920 dispatch_meta_bytes=887752 combine_payload_bytes=905297920 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=906185672 ingress_bytes=1257619456 link_bytes=1257619456 token_bytes=1811483592 weight_movement_bytes=352321536 total_nvlink_bytes=2163805128 weight_to_token_ratio=0.194493 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=18048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.042143997e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 6, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917304 combine_payload_bytes=939343872 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940261176 ingress_bytes=1203585024 link_bytes=1203585024 token_bytes=1879605048 weight_movement_bytes=264241152 total_nvlink_bytes=2143846200 weight_to_token_ratio=0.140583 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017045 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739573733e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130966 payload_rows_remote=114622 dup_rows=131178 dup_rows_remote=117178 dedup_ratio=0.500404 router_tokens_per_expert=[0, 11, 8138, 25, 8155, 9, 0, 7377, 8192, 8130, 0, 0, 0, 13, 8187, 3, 0, 6893, 3, 1, 8175, 8132, 143, 0, 1101, 1, 1, 24, 0, 8015, 0, 0, 8184, 11, 1, 8148, 56, 31, 0, 0, 18, 0, 1, 833, 0, 85, 0, 147, 8187, 0, 8171, 0, 4, 65, 1, 9, 8191, 0, 0, 0, 8190, 0, 3, 7] payload_rows_per_dst=[16334, 16372, 16344, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[14786, 16268, 14000, 18210, 16478, 18544, 16494, 16398] combine_rows_per_src=[16334, 16372, 16344, 16384, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114058 dedup_ratio=0.500084 router_tokens_per_expert=[0, 29, 8177, 0, 8186, 5, 7, 7951, 8190, 8189, 1, 0, 5, 1, 8171, 0, 0, 6586, 0, 0, 8190, 8151, 11, 1, 316, 3, 18, 1, 0, 8185, 1, 0, 8190, 2, 0, 8188, 5, 0, 1, 0, 2, 3, 1, 649, 2, 10, 1, 41, 8177, 1, 8191, 0, 2, 833, 0, 5, 8191, 0, 0, 4, 8188, 0, 8, 2] payload_rows_per_dst=[16384, 16382, 16364, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15954, 16390, 13134, 17036, 16388, 17756, 18034, 16402] combine_rows_per_src=[16384, 16382, 16364, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130874 payload_rows_remote=114490 dup_rows=131270 dup_rows_remote=111728 dedup_ratio=0.500755 router_tokens_per_expert=[1, 71, 8153, 1, 8155, 1, 2, 7462, 8187, 8180, 1, 5, 16, 0, 8066, 0, 0, 8086, 23, 4, 8088, 7180, 173, 0, 91, 6, 1, 0, 15, 8179, 3, 0, 8142, 0, 2, 8168, 51, 19, 22, 0, 2, 54, 16, 1278, 14, 20, 13, 215, 8139, 0, 8171, 3, 5, 210, 5, 5, 8175, 1, 0, 0, 8171, 10, 11, 0] payload_rows_per_dst=[16380, 16242, 16380, 16374, 16384, 16384, 16360, 16370] dup_rows_per_dst=[15002, 16142, 14552, 16526, 16424, 19542, 16716, 16366] combine_rows_per_src=[16380, 16242, 16380, 16374, 16384, 16384, 16360, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130760 payload_rows_remote=114376 dup_rows=131384 dup_rows_remote=114862 dedup_ratio=0.501190 router_tokens_per_expert=[0, 0, 8170, 0, 8191, 3, 0, 7334, 8190, 8189, 0, 0, 0, 0, 8192, 0, 0, 6575, 0, 0, 8179, 7061, 149, 0, 1859, 1, 1, 0, 0, 8175, 0, 0, 8191, 0, 1, 8173, 0, 11, 0, 0, 1, 0, 1, 1341, 1, 48, 0, 182, 8142, 2, 8191, 0, 1, 117, 0, 0, 8192, 0, 9, 0, 8191, 1, 7, 0] payload_rows_per_dst=[16372, 16378, 16090, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14642, 6682, 21156, 20070, 16368, 19528, 16522, 16416] combine_rows_per_src=[16372, 16378, 16090, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130194 payload_rows_remote=113810 dup_rows=131950 dup_rows_remote=105340 dedup_ratio=0.503349 router_tokens_per_expert=[0, 22, 8090, 0, 8177, 26, 118, 7120, 8192, 8176, 0, 0, 4, 3, 8192, 0, 1, 7280, 2, 0, 8157, 5620, 398, 4, 218, 0, 0, 0, 0, 7834, 0, 0, 8173, 10, 2, 8191, 0, 86, 1, 0, 0, 0, 0, 2253, 4, 253, 0, 1283, 7944, 0, 8191, 0, 1, 607, 0, 0, 8184, 1, 4, 0, 8190, 0, 59, 1] payload_rows_per_dst=[16368, 16384, 15552, 16354, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16200, 26610, 11058, 14288, 16542, 13656, 17102, 16494] combine_rows_per_src=[16368, 16384, 15552, 16354, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114666 dedup_ratio=0.500031 router_tokens_per_expert=[0, 1, 8180, 0, 8189, 0, 0, 7891, 8187, 8189, 0, 0, 0, 0, 8189, 0, 0, 7646, 0, 0, 8184, 4783, 35, 0, 1208, 2, 2, 1, 0, 8188, 1, 0, 8191, 0, 0, 8191, 0, 125, 0, 0, 2, 2, 0, 2029, 0, 14, 0, 386, 8188, 0, 8191, 0, 0, 490, 0, 0, 8192, 0, 1, 2, 8191, 0, 1, 0] payload_rows_per_dst=[16384, 16378, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15760, 0, 24914, 18798, 16600, 21234, 17360, 16414] combine_rows_per_src=[16384, 16378, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 2, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=927200 combine_payload_bytes=938983424 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939910624 ingress_bytes=1291304960 link_bytes=1291304960 token_bytes=1878894048 weight_movement_bytes=352321536 total_nvlink_bytes=2231215584 weight_to_token_ratio=0.187515 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230768759e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=914896 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940258768 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879602640 weight_movement_bytes=176160768 total_nvlink_bytes=2055763408 weight_to_token_ratio=0.093722 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114684 dup_rows=131078 dup_rows_remote=114796 dedup_ratio=0.500023 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 8191, 16, 21, 8129, 1, 8180, 8187, 8178, 0, 6, 1, 0, 8175, 1, 8184, 1, 11, 8094, 12, 0, 0, 304, 7, 0, 1, 8190, 8157, 20, 0, 0, 2, 0, 8153, 1, 3, 0, 8179, 8191, 0, 1, 10, 1, 8192, 1, 0, 0, 0, 17, 0, 1, 1, 1, 8059, 0, 8191, 0, 0] payload_rows_per_dst=[16382, 16382, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16368, 16322, 16374, 16828, 16282, 16386, 16396, 16122] combine_rows_per_src=[16382, 16382, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=937902080 dispatch_meta_bytes=904872 combine_payload_bytes=937902080 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938806952 ingress_bytes=1290223616 link_bytes=1290223616 token_bytes=1876709032 weight_movement_bytes=352321536 total_nvlink_bytes=2229030568 weight_to_token_ratio=0.187734 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.019886 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224713684e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=936968192 dispatch_meta_bytes=916952 combine_payload_bytes=936968192 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=937885144 ingress_bytes=1465450496 link_bytes=1465450496 token_bytes=1874853336 weight_movement_bytes=528482304 total_nvlink_bytes=2403335640 weight_to_token_ratio=0.281879 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.205911068e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130158 payload_rows_remote=113774 dup_rows=131986 dup_rows_remote=107494 dedup_ratio=0.503487 router_tokens_per_expert=[0, 22, 8192, 0, 8185, 0, 0, 5062, 8187, 8189, 4, 4, 0, 3, 8187, 0, 3, 8136, 5, 2, 8190, 7710, 89, 2, 1840, 0, 0, 0, 2, 8189, 0, 0, 8190, 0, 1, 8189, 0, 15, 0, 3, 0, 0, 0, 1049, 2, 27, 0, 377, 8126, 4, 8189, 2, 2, 308, 1, 6, 8189, 0, 1, 0, 8188, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16378, 15482, 16378] dup_rows_per_dst=[24492, 32770, 19038, 8084, 15882, 2906, 12618, 16196] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16378, 15482, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=932331520 dispatch_meta_bytes=876600 combine_payload_bytes=932331520 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=933208120 ingress_bytes=1460813824 link_bytes=1460813824 token_bytes=1865539640 weight_movement_bytes=528482304 total_nvlink_bytes=2394021944 weight_to_token_ratio=0.283287 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.179947641e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917384 combine_payload_bytes=939458560 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940375944 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834504 weight_movement_bytes=176160768 total_nvlink_bytes=2055995272 weight_to_token_ratio=0.093711 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917920 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940409248 ingress_bytes=1379893248 link_bytes=1379893248 token_bytes=1879900576 weight_movement_bytes=440401920 total_nvlink_bytes=2320302496 weight_to_token_ratio=0.234269 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726826193e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=932036608 dispatch_meta_bytes=885072 combine_payload_bytes=932036608 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=932921680 ingress_bytes=1284358144 link_bytes=1284358144 token_bytes=1864958288 weight_movement_bytes=352321536 total_nvlink_bytes=2217279824 weight_to_token_ratio=0.188917 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.191869489e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114750 dedup_ratio=0.500084 router_tokens_per_expert=[8189, 0, 0, 8137, 2, 14, 0, 8191, 3, 8190, 0, 8171, 0, 3, 13, 0, 1, 8170, 6, 64, 0, 8180, 0, 0, 8190, 2, 0, 0, 0, 0, 8172, 0, 8179, 7939, 1, 0, 3, 2, 0, 0, 2, 0, 2, 40, 8189, 1, 0, 1, 12, 209, 0, 2, 1, 8190, 8190, 0, 5, 3, 60, 0, 0, 8156, 0, 8187] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16366, 16384, 16384, 16384] dup_rows_per_dst=[16304, 16376, 16458, 16344, 15882, 16468, 16824, 16438] combine_rows_per_src=[16380, 16384, 16384, 16384, 16366, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114662 dup_rows=131102 dup_rows_remote=114696 dedup_ratio=0.500114 router_tokens_per_expert=[8180, 0, 0, 8019, 4, 1, 0, 8165, 27, 8190, 0, 7974, 0, 7, 255, 0, 2, 8190, 0, 26, 0, 8138, 0, 37, 8192, 0, 0, 0, 0, 0, 8130, 0, 7601, 8184, 0, 0, 0, 1, 2, 0, 0, 0, 8, 17, 8129, 9, 3, 70, 25, 674, 0, 10, 0, 8187, 8192, 0, 61, 0, 31, 6, 0, 8189, 3, 8133] payload_rows_per_dst=[16358, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16040, 16522, 16406, 16260, 15192, 16428, 17792, 16462] combine_rows_per_src=[16358, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114670 dup_rows=131102 dup_rows_remote=114708 dedup_ratio=0.500114 router_tokens_per_expert=[8093, 1, 19, 8123, 4, 258, 9, 8158, 80, 8176, 14, 7937, 3, 0, 20, 6, 8, 8126, 14, 18, 7, 8178, 21, 1, 8179, 3, 1, 0, 1, 13, 8183, 25, 8179, 8023, 5, 2, 36, 0, 6, 1, 0, 13, 8, 1, 8168, 11, 0, 10, 0, 19, 15, 1, 2, 8180, 8177, 1, 2, 9, 166, 5, 0, 8167, 6, 8180] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16378, 16372, 16384, 16376] dup_rows_per_dst=[16602, 16092, 16362, 16426, 16126, 16394, 16406, 16694] combine_rows_per_src=[16384, 16380, 16384, 16384, 16378, 16372, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114588 dedup_ratio=0.500061 router_tokens_per_expert=[8175, 0, 0, 8126, 2, 1, 1, 8191, 41, 8190, 0, 8157, 1, 0, 13, 1, 2, 8191, 0, 41, 2, 8190, 0, 0, 8190, 0, 1, 0, 0, 0, 8192, 0, 8171, 8138, 1, 1, 0, 0, 2, 0, 1, 2, 0, 0, 8190, 0, 0, 23, 3, 56, 1, 1, 0, 8190, 8191, 0, 0, 0, 14, 0, 0, 8189, 0, 8190] payload_rows_per_dst=[16372, 16384, 16384, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16258, 16422, 16468, 16382, 16242, 16410, 16500, 16406] combine_rows_per_src=[16372, 16384, 16384, 16384, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917664 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940261536 ingress_bytes=1291665408 link_bytes=1291665408 token_bytes=1879605408 weight_movement_bytes=352321536 total_nvlink_bytes=2231926944 weight_to_token_ratio=0.187444 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232787117e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114390 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 8174, 0, 8, 0, 8187, 8, 8191, 1, 8155, 0, 0, 45, 0, 2, 8184, 0, 17, 0, 8184, 2, 0, 8190, 0, 0, 6, 0, 0, 8188, 0, 8182, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 2, 2, 1, 10, 0, 0, 3, 8191, 8192, 2, 0, 0, 7, 0, 0, 8189, 1, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16052, 16582, 16230, 16384, 16332, 16382, 16428, 16682] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114684 dup_rows=131104 dup_rows_remote=114856 dedup_ratio=0.500122 router_tokens_per_expert=[8192, 0, 0, 8111, 60, 177, 0, 8121, 1, 8178, 0, 8120, 0, 0, 3, 0, 0, 8152, 0, 53, 0, 8171, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 7862, 4, 0, 0, 0, 12, 0, 0, 0, 0, 59, 8169, 0, 2, 126, 1, 79, 0, 0, 4, 8192, 8192, 0, 12, 0, 60, 0, 1, 8191, 0, 8191] payload_rows_per_dst=[16384, 16356, 16382, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16696, 16248, 16370, 16384, 15756, 16570, 16552, 16528] combine_rows_per_src=[16384, 16356, 16382, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=917432 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=940228536 ingress_bytes=1203552256 link_bytes=1203552256 token_bytes=1879539640 weight_movement_bytes=264241152 total_nvlink_bytes=2143780792 weight_to_token_ratio=0.140588 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739390246e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917512 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940294152 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879670792 weight_movement_bytes=176160768 total_nvlink_bytes=2055831560 weight_to_token_ratio=0.093719 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130854 payload_rows_remote=114472 dup_rows=131290 dup_rows_remote=114898 dedup_ratio=0.500832 router_tokens_per_expert=[0, 2, 8160, 0, 8190, 0, 0, 7189, 8191, 8192, 0, 0, 1, 0, 8188, 0, 0, 7652, 0, 0, 8192, 7536, 210, 0, 860, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 1, 8188, 1, 6, 0, 0, 0, 0, 1, 594, 0, 60, 0, 110, 8173, 1, 8192, 0, 0, 403, 0, 1, 8192, 0, 6, 0, 8192, 0, 5, 0] payload_rows_per_dst=[16380, 16384, 16172, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[14322, 16378, 14624, 18100, 16392, 17912, 17156, 16406] combine_rows_per_src=[16380, 16384, 16172, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114674 dup_rows=131088 dup_rows_remote=114308 dedup_ratio=0.500061 router_tokens_per_expert=[8188, 2, 0, 8160, 0, 2, 2, 8189, 23, 8189, 0, 8170, 1, 0, 3, 2, 0, 8167, 0, 2, 5, 8192, 3, 0, 8189, 2, 2, 11, 3, 4, 8184, 1, 8117, 8157, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 23, 3, 82, 3, 0, 7, 8184, 8188, 0, 0, 3, 25, 2, 0, 8189, 0, 8190] payload_rows_per_dst=[16382, 16384, 16384, 16374, 16382, 16382, 16384, 16384] dup_rows_per_dst=[16780, 16392, 16380, 16510, 19608, 16000, 13954, 15464] combine_rows_per_src=[16382, 16384, 16384, 16374, 16382, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917040 combine_payload_bytes=939393024 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940310064 ingress_bytes=1291714560 link_bytes=1291714560 token_bytes=1879703088 weight_movement_bytes=352321536 total_nvlink_bytes=2232024624 weight_to_token_ratio=0.187435 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233062348e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916312 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940440408 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879964504 weight_movement_bytes=352321536 total_nvlink_bytes=2232286040 weight_to_token_ratio=0.187409 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=918160 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=940409488 ingress_bytes=1203732480 link_bytes=1203732480 token_bytes=1879900816 weight_movement_bytes=264241152 total_nvlink_bytes=2144141968 weight_to_token_ratio=0.140561 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740399425e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=915928 combine_payload_bytes=939409408 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940325336 ingress_bytes=1467891712 link_bytes=1467891712 token_bytes=1879734744 weight_movement_bytes=528482304 total_nvlink_bytes=2408217048 weight_to_token_ratio=0.281147 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219580859e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=937754624 dispatch_meta_bytes=917480 combine_payload_bytes=937754624 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=938672104 ingress_bytes=1290076160 link_bytes=1290076160 token_bytes=1876426728 weight_movement_bytes=352321536 total_nvlink_bytes=2228748264 weight_to_token_ratio=0.187762 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.223887992e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114746 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 6, 8189, 8154, 0, 0, 8191, 0, 8171, 0, 1, 0, 8076, 3, 0, 0, 8165, 6, 0, 8187, 0, 0, 2, 4, 8192, 164, 5, 2, 5, 8134, 8190, 41, 2, 0, 0, 8190, 0, 0, 0, 0, 8191, 0, 1, 36, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 8192, 0] payload_rows_per_dst=[16380, 16380, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16316, 16120, 16336, 16628, 16462, 16456, 16384] combine_rows_per_src=[16380, 16380, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131012 payload_rows_remote=114654 dup_rows=131132 dup_rows_remote=115248 dedup_ratio=0.500229 router_tokens_per_expert=[0, 20, 1, 8192, 0, 0, 0, 0, 88, 0, 0, 8050, 8161, 0, 0, 8147, 2, 8179, 0, 0, 7, 7933, 0, 0, 0, 8165, 0, 0, 8180, 0, 0, 1, 95, 8192, 72, 0, 14, 12, 8133, 8191, 41, 0, 2, 0, 8192, 0, 9, 0, 0, 8190, 0, 0, 257, 0, 0, 0, 8173, 0, 3, 0, 8188, 1, 8181, 0] payload_rows_per_dst=[16384, 16372, 16358, 16380, 16384, 16384, 16380, 16370] dup_rows_per_dst=[16422, 16196, 15884, 16312, 16652, 16482, 16838, 16346] combine_rows_per_src=[16384, 16372, 16358, 16380, 16384, 16384, 16380, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131004 payload_rows_remote=114622 dup_rows=131140 dup_rows_remote=114750 dedup_ratio=0.500259 router_tokens_per_expert=[15, 0, 5, 8174, 10, 22, 20, 6, 0, 16, 8, 8161, 8160, 13, 0, 8152, 0, 8149, 15, 11, 32, 8148, 0, 12, 0, 8117, 0, 15, 8148, 33, 0, 0, 32, 8171, 31, 13, 2, 6, 8157, 8177, 0, 7, 3, 12, 8168, 3, 4, 12, 29, 8154, 2, 68, 0, 0, 1, 12, 8161, 22, 19, 14, 8174, 5, 8169, 2] payload_rows_per_dst=[16384, 16372, 16380, 16366, 16352, 16382, 16384, 16384] dup_rows_per_dst=[16468, 16294, 16354, 16260, 16472, 16390, 16502, 16400] combine_rows_per_src=[16384, 16372, 16380, 16366, 16352, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917696 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940359872 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879802048 weight_movement_bytes=176160768 total_nvlink_bytes=2055962816 weight_to_token_ratio=0.093712 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114680 dup_rows=131082 dup_rows_remote=114118 dedup_ratio=0.500038 router_tokens_per_expert=[0, 6, 2, 8192, 0, 0, 0, 1, 0, 1, 0, 8191, 8188, 0, 0, 8179, 0, 8175, 0, 0, 0, 8140, 2, 0, 0, 8191, 2, 0, 8190, 0, 0, 0, 31, 8192, 52, 4, 0, 0, 8177, 8192, 6, 1, 4, 0, 8192, 0, 0, 0, 0, 8184, 0, 0, 3, 0, 0, 2, 8191, 0, 0, 0, 8191, 0, 8190, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16380, 16382] dup_rows_per_dst=[16128, 16192, 16254, 16548, 16698, 15924, 16374, 16964] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16380, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114622 dedup_ratio=0.500084 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 1, 1, 0, 0, 0, 8181, 8181, 0, 0, 8172, 4, 8179, 0, 0, 5, 8012, 0, 1, 0, 8191, 0, 0, 8190, 0, 0, 2, 42, 8191, 91, 1, 7, 0, 8181, 8191, 46, 0, 0, 0, 8191, 0, 1, 0, 1, 8188, 0, 1, 50, 0, 0, 0, 8192, 1, 0, 0, 8192, 0, 8192, 1] payload_rows_per_dst=[16384, 16366, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 16326, 16022, 16382, 16642, 16476, 16472, 16388] combine_rows_per_src=[16384, 16366, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130962 payload_rows_remote=114578 dup_rows=131182 dup_rows_remote=114936 dedup_ratio=0.500420 router_tokens_per_expert=[0, 42, 16, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8173, 0, 0, 8133, 0, 8104, 0, 0, 0, 7598, 0, 0, 3, 8178, 0, 0, 8192, 0, 0, 0, 203, 8192, 476, 12, 0, 1, 8065, 8192, 108, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 7, 38, 0, 0, 0, 8189, 0, 0, 0, 8192, 0, 8191, 0] payload_rows_per_dst=[16384, 16384, 16276, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16500, 16246, 15128, 16362, 17514, 16598, 16456, 16378] combine_rows_per_src=[16384, 16384, 16276, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=919608 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940165176 ingress_bytes=1115406336 link_bytes=1115406336 token_bytes=1879410744 weight_movement_bytes=176160768 total_nvlink_bytes=2055571512 weight_to_token_ratio=0.093732 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245809888e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114644 dup_rows=131116 dup_rows_remote=114708 dedup_ratio=0.500168 router_tokens_per_expert=[3, 6, 0, 8189, 0, 2, 4, 3, 0, 3, 43, 8185, 8187, 3, 0, 8185, 0, 8146, 0, 10, 0, 8184, 0, 2, 0, 8189, 0, 2, 8186, 5, 0, 1, 2, 8188, 1, 0, 0, 14, 8189, 8188, 0, 4, 0, 4, 8188, 3, 2, 0, 0, 8171, 0, 0, 0, 0, 0, 1, 8189, 4, 7, 0, 8189, 2, 8183, 5] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16344, 16384] dup_rows_per_dst=[16408, 17210, 19122, 16516, 14078, 16296, 15640, 15846] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16344, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=917488 combine_payload_bytes=938983424 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939900912 ingress_bytes=1115144192 link_bytes=1115144192 token_bytes=1878884336 weight_movement_bytes=176160768 total_nvlink_bytes=2055045104 weight_to_token_ratio=0.093758 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244341991e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=115128 dedup_ratio=0.500061 router_tokens_per_expert=[8190, 0, 0, 8141, 0, 59, 0, 8184, 254, 8191, 0, 7965, 0, 0, 17, 0, 1, 8191, 3, 27, 0, 8172, 0, 0, 8192, 0, 0, 5, 0, 0, 8191, 0, 7971, 8155, 35, 0, 10, 0, 1, 0, 0, 0, 0, 12, 8192, 0, 0, 15, 0, 114, 0, 0, 3, 8191, 8192, 0, 2, 0, 14, 0, 0, 8192, 0, 8190] payload_rows_per_dst=[16368, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16400, 16470, 16404, 16392, 15960, 16434, 16616, 16412] combine_rows_per_src=[16368, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=915192 combine_payload_bytes=939458560 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940373752 ingress_bytes=1379860480 link_bytes=1379860480 token_bytes=1879832312 weight_movement_bytes=440401920 total_nvlink_bytes=2320234232 weight_to_token_ratio=0.234277 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726642706e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917152 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940261024 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879604896 weight_movement_bytes=176160768 total_nvlink_bytes=2055765664 weight_to_token_ratio=0.093722 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938622976 dispatch_meta_bytes=918056 combine_payload_bytes=938622976 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=939541032 ingress_bytes=1290944512 link_bytes=1290944512 token_bytes=1878164008 weight_movement_bytes=352321536 total_nvlink_bytes=2230485544 weight_to_token_ratio=0.187588 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228750400e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=917408 combine_payload_bytes=939163648 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940081056 ingress_bytes=1115324416 link_bytes=1115324416 token_bytes=1879244704 weight_movement_bytes=176160768 total_nvlink_bytes=2055405472 weight_to_token_ratio=0.093740 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245351170e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114848 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8190, 0, 0, 0, 1, 12, 3, 8122, 8190, 8192, 2, 64, 8189, 1, 6328, 1, 1, 0, 8188, 0, 1, 525, 0, 67, 0, 0, 0, 8047, 0, 8191, 1, 92, 8, 8159, 0, 0, 0, 8101, 0, 0, 0, 8191, 14, 0, 8192, 0, 1, 995, 1, 8192, 435, 1, 0, 0, 1, 0, 0, 0, 0, 8190, 2, 0, 8181] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16382] dup_rows_per_dst=[16406, 12650, 17428, 16228, 16338, 16412, 19250, 16364] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=919200 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940312224 ingress_bytes=1115553792 link_bytes=1115553792 token_bytes=1879705248 weight_movement_bytes=176160768 total_nvlink_bytes=2055866016 weight_to_token_ratio=0.093717 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.018939 gemm_active_groups=17 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246635580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130930 payload_rows_remote=114546 dup_rows=131214 dup_rows_remote=113980 dedup_ratio=0.500542 router_tokens_per_expert=[0, 8184, 3, 0, 0, 0, 0, 0, 7817, 8192, 8192, 0, 97, 8147, 6, 5858, 0, 0, 0, 8192, 0, 0, 780, 0, 53, 0, 1, 0, 8163, 0, 8192, 0, 496, 8, 8131, 0, 0, 0, 7911, 1, 0, 0, 8192, 204, 0, 8179, 0, 21, 30, 0, 8192, 1574, 0, 0, 0, 0, 0, 0, 0, 19, 8191, 3, 0, 8043] payload_rows_per_dst=[16384, 16254, 16384, 16384, 16374, 16384, 16384, 16382] dup_rows_per_dst=[16374, 11922, 17234, 16434, 16720, 16808, 19592, 16130] combine_rows_per_src=[16384, 16254, 16384, 16384, 16374, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=918128 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=940409456 ingress_bytes=1203732480 link_bytes=1203732480 token_bytes=1879900784 weight_movement_bytes=264241152 total_nvlink_bytes=2144141936 weight_to_token_ratio=0.140561 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740399425e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114666 dup_rows=131120 dup_rows_remote=114428 dedup_ratio=0.500183 router_tokens_per_expert=[15, 8130, 1, 16, 16, 4, 2, 25, 8178, 8124, 8156, 18, 23, 8172, 21, 2919, 0, 4, 14, 8154, 0, 31, 2257, 13, 19, 39, 0, 0, 8166, 30, 8159, 25, 907, 0, 8096, 7, 0, 5, 8098, 0, 0, 0, 8164, 96, 0, 8174, 3, 88, 498, 19, 8024, 2053, 0, 0, 7, 18, 0, 20, 17, 0, 8150, 0, 0, 7897] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16358, 16384, 16366] dup_rows_per_dst=[16384, 5934, 20804, 16492, 17846, 16692, 21166, 15802] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16358, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=113206 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8192, 1, 0, 0, 0, 0, 0, 8011, 8190, 8192, 0, 64, 8192, 0, 3539, 0, 0, 0, 8190, 1, 0, 4245, 0, 52, 0, 0, 0, 8154, 0, 8192, 0, 0, 1, 8191, 2, 0, 0, 8132, 0, 0, 0, 8192, 27, 0, 8190, 2, 2, 2, 0, 8190, 751, 0, 1, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 7981] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16374, 6834, 24872, 15764, 16136, 15342, 17884, 17870] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=105124 dedup_ratio=0.500031 router_tokens_per_expert=[1, 8192, 0, 0, 0, 0, 1, 0, 7811, 8192, 8192, 0, 103, 8176, 0, 3994, 0, 0, 0, 8192, 2, 0, 4529, 0, 248, 0, 0, 0, 8012, 0, 8192, 0, 52, 2, 8192, 0, 0, 0, 8065, 0, 0, 0, 8192, 13, 0, 8190, 0, 0, 7, 0, 8187, 160, 0, 0, 0, 0, 0, 0, 0, 14, 8192, 0, 0, 7969] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16376, 16384] dup_rows_per_dst=[16356, 25956, 12920, 16520, 16238, 16406, 10718, 15966] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131026 payload_rows_remote=114642 dup_rows=131118 dup_rows_remote=112858 dedup_ratio=0.500175 router_tokens_per_expert=[0, 8187, 0, 0, 0, 1, 0, 0, 8134, 8190, 8191, 0, 36, 8189, 0, 7077, 0, 1, 0, 8191, 0, 1, 404, 1, 48, 0, 0, 0, 8192, 1, 8191, 0, 2, 0, 8168, 0, 0, 0, 8018, 1, 0, 0, 8191, 4, 0, 8192, 0, 4, 8, 0, 8190, 932, 0, 0, 1, 1, 1, 1, 1, 1, 8191, 0, 0, 8130] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16338, 16384, 16384, 16384] dup_rows_per_dst=[16374, 14110, 17190, 16480, 16040, 16398, 18260, 16266] combine_rows_per_src=[16384, 16384, 16384, 16384, 16338, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938360832 dispatch_meta_bytes=914104 combine_payload_bytes=938360832 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939274936 ingress_bytes=1114521600 link_bytes=1114521600 token_bytes=1877635768 weight_movement_bytes=176160768 total_nvlink_bytes=2053796536 weight_to_token_ratio=0.093821 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.240855736e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130706 payload_rows_remote=114322 dup_rows=131438 dup_rows_remote=115016 dedup_ratio=0.501396 router_tokens_per_expert=[13, 8182, 0, 5, 2, 0, 0, 0, 8145, 8187, 8177, 0, 124, 8188, 0, 4415, 1, 5, 0, 8181, 7, 6, 1468, 1, 8, 8, 0, 0, 8165, 3, 8186, 6, 345, 9, 8139, 2, 0, 3, 7921, 0, 4, 1, 8186, 0, 0, 8189, 0, 18, 40, 2, 8186, 2191, 0, 2, 0, 4, 0, 3, 8, 0, 8184, 0, 0, 8152] payload_rows_per_dst=[16384, 16374, 16376, 16384, 16042, 16382, 16380, 16384] dup_rows_per_dst=[16422, 42848, 2962, 16784, 15228, 16414, 4470, 16310] combine_rows_per_src=[16384, 16374, 16376, 16384, 16042, 16382, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[3, 0, 2, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=916376 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940260248 ingress_bytes=1291665408 link_bytes=1291665408 token_bytes=1879604120 weight_movement_bytes=352321536 total_nvlink_bytes=2231925656 weight_to_token_ratio=0.187445 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232787117e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114650 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 8170, 8188, 0, 0, 8192, 1, 8171, 0, 0, 0, 8187, 1, 0, 0, 8189, 0, 0, 8192, 0, 0, 0, 6, 8192, 11, 0, 2, 1, 8192, 8192, 31, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16340, 16336, 16378, 16424, 16446, 16382, 16384] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=911560 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940402888 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879894216 weight_movement_bytes=176160768 total_nvlink_bytes=2056054984 weight_to_token_ratio=0.093708 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 6, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=879216 combine_payload_bytes=939458560 prefetch_bytes=88080384 reduce_bytes=704643072 egress_bytes=940337776 ingress_bytes=1732182016 link_bytes=1732182016 token_bytes=1879796336 weight_movement_bytes=792723456 total_nvlink_bytes=2672519792 weight_to_token_ratio=0.421707 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.022727 gemm_active_groups=17 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699496241e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=910000 combine_payload_bytes=939147264 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940057264 ingress_bytes=1115308032 link_bytes=1115308032 token_bytes=1879204528 weight_movement_bytes=176160768 total_nvlink_bytes=2055365296 weight_to_token_ratio=0.093742 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245259427e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 2, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130918 payload_rows_remote=114534 dup_rows=131226 dup_rows_remote=103574 dedup_ratio=0.500587 router_tokens_per_expert=[6905, 0, 0, 0, 1526, 0, 8189, 0, 0, 0, 400, 2, 0, 0, 5888, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8191, 1, 8191, 5633, 2, 0, 8192, 8192, 0, 1, 0, 0, 3, 8188, 2, 4, 0, 8192, 7, 4097, 0, 0, 8075, 0, 47, 0, 0, 8190, 0, 0, 8191, 106, 0, 0, 55, 8192, 8176, 44, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16236, 16382, 16384] dup_rows_per_dst=[16860, 12580, 16380, 27652, 16388, 8130, 16474, 16762] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16236, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=936525824 dispatch_meta_bytes=917352 combine_payload_bytes=936525824 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=937443176 ingress_bytes=1200766976 link_bytes=1200766976 token_bytes=1873969000 weight_movement_bytes=264241152 total_nvlink_bytes=2138210152 weight_to_token_ratio=0.141006 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.723793841e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917344 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940425056 ingress_bytes=1556070400 link_bytes=1556070400 token_bytes=1879932768 weight_movement_bytes=616562688 total_nvlink_bytes=2496495456 weight_to_token_ratio=0.327971 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713344704e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 4, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130902 payload_rows_remote=114518 dup_rows=131242 dup_rows_remote=114846 dedup_ratio=0.500648 router_tokens_per_expert=[4415, 9, 0, 0, 2435, 0, 8191, 0, 0, 15, 508, 0, 5, 0, 5376, 0, 0, 5, 0, 8192, 0, 0, 0, 1, 0, 8187, 0, 8191, 4576, 1, 0, 8183, 8192, 0, 0, 0, 0, 0, 8179, 13, 0, 0, 8192, 0, 6360, 0, 0, 7822, 0, 1062, 0, 0, 8192, 0, 0, 8187, 9, 0, 0, 157, 8192, 8185, 26, 14] payload_rows_per_dst=[16384, 16384, 16384, 16374, 16384, 16224, 16384, 16384] dup_rows_per_dst=[13716, 11806, 16396, 25520, 16384, 12140, 18498, 16782] combine_rows_per_src=[16384, 16384, 16384, 16374, 16384, 16224, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938262528 dispatch_meta_bytes=872432 combine_payload_bytes=938262528 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939134960 ingress_bytes=1642905600 link_bytes=1642905600 token_bytes=1877397488 weight_movement_bytes=704643072 total_nvlink_bytes=2582040560 weight_to_token_ratio=0.375330 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=28672 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.199585577e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131026 payload_rows_remote=114642 dup_rows=131118 dup_rows_remote=107320 dedup_ratio=0.500175 router_tokens_per_expert=[6669, 18, 12, 15, 4058, 21, 8171, 0, 22, 0, 103, 24, 56, 24, 5096, 25, 0, 0, 17, 8065, 5, 25, 6, 0, 24, 8162, 0, 8166, 4779, 2, 21, 8167, 8127, 0, 33, 1, 20, 1, 8171, 0, 0, 4, 8152, 0, 3766, 20, 36, 8099, 0, 128, 0, 12, 8150, 0, 2, 8171, 100, 10, 24, 1, 8169, 8104, 18, 0] payload_rows_per_dst=[16384, 16384, 16360, 16382, 16382, 16384, 16366, 16384] dup_rows_per_dst=[21544, 10650, 16180, 9594, 16324, 23798, 16560, 16468] combine_rows_per_src=[16384, 16384, 16360, 16382, 16382, 16384, 16366, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130980 payload_rows_remote=114596 dup_rows=131164 dup_rows_remote=110262 dedup_ratio=0.500351 router_tokens_per_expert=[2899, 0, 0, 0, 4860, 0, 8192, 0, 0, 0, 112, 0, 1, 0, 7865, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 8192, 0, 8190, 2532, 1, 0, 8192, 8186, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 8192, 8, 3958, 0, 0, 8090, 0, 1261, 1, 0, 8191, 2, 0, 8192, 1134, 0, 0, 20, 8192, 8192, 30, 2] payload_rows_per_dst=[16292, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[7418, 15952, 16384, 13258, 16372, 24114, 16764, 20902] combine_rows_per_src=[16292, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130964 payload_rows_remote=114580 dup_rows=131180 dup_rows_remote=109316 dedup_ratio=0.500412 router_tokens_per_expert=[3886, 0, 0, 0, 5053, 0, 8192, 0, 0, 0, 2265, 0, 0, 0, 7275, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 845, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 1, 3842, 0, 0, 8137, 0, 280, 0, 0, 8192, 0, 0, 8192, 1070, 0, 0, 114, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16276, 16384, 16384] dup_rows_per_dst=[17878, 21864, 16384, 15290, 16384, 7684, 16944, 18752] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16276, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130650 payload_rows_remote=114266 dup_rows=131494 dup_rows_remote=114650 dedup_ratio=0.501610 router_tokens_per_expert=[6986, 1, 0, 0, 931, 1, 8191, 2, 0, 2, 147, 1, 0, 0, 7204, 1, 0, 1, 0, 8188, 1, 0, 0, 0, 2, 8191, 0, 8190, 4431, 0, 0, 8190, 8186, 0, 0, 0, 1, 0, 8190, 5, 0, 0, 8188, 6, 4503, 0, 0, 8156, 2, 227, 3, 0, 8191, 0, 0, 8191, 17, 1, 1, 162, 8190, 8191, 0, 0] payload_rows_per_dst=[15966, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[10048, 14704, 16376, 15074, 16380, 25328, 16844, 16740] combine_rows_per_src=[15966, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938131456 dispatch_meta_bytes=917456 combine_payload_bytes=938131456 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939048912 ingress_bytes=1290452992 link_bytes=1290452992 token_bytes=1877180368 weight_movement_bytes=352321536 total_nvlink_bytes=2229501904 weight_to_token_ratio=0.187687 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.225998094e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130188 payload_rows_remote=113804 dup_rows=131956 dup_rows_remote=103164 dedup_ratio=0.503372 router_tokens_per_expert=[6636, 3, 0, 3, 2829, 6, 8186, 1, 6, 0, 229, 0, 0, 5, 7773, 6, 0, 0, 2, 8173, 4, 5, 0, 0, 0, 8186, 0, 8187, 2120, 0, 0, 8187, 8184, 3, 14, 5, 3, 0, 8187, 0, 10, 13, 8176, 2, 4792, 5, 5, 8164, 0, 23, 0, 1, 8182, 0, 6, 8186, 2, 4, 5, 222, 8184, 8147, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16152, 16384, 15736] dup_rows_per_dst=[28792, 32400, 16612, 7196, 16460, 9554, 12244, 8698] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16152, 16384, 15736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=887848 combine_payload_bytes=939147264 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940035112 ingress_bytes=1467629568 link_bytes=1467629568 token_bytes=1879182376 weight_movement_bytes=528482304 total_nvlink_bytes=2407664680 weight_to_token_ratio=0.281230 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218112962e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114142 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8101, 8187, 8192, 0, 34, 8191, 0, 5538, 0, 0, 0, 8192, 0, 0, 643, 0, 27, 0, 0, 0, 8158, 0, 8192, 0, 304, 1, 8192, 0, 0, 0, 8160, 0, 0, 0, 8192, 117, 0, 8192, 0, 5, 336, 0, 8189, 1352, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 10962, 17670, 16370, 16930, 16628, 19744, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938770432 dispatch_meta_bytes=899432 combine_payload_bytes=938770432 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=939669864 ingress_bytes=1379172352 link_bytes=1379172352 token_bytes=1878440296 weight_movement_bytes=440401920 total_nvlink_bytes=2318842216 weight_to_token_ratio=0.234451 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.722789476e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938639360 dispatch_meta_bytes=895584 combine_payload_bytes=938639360 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939534944 ingress_bytes=1290960896 link_bytes=1290960896 token_bytes=1878174304 weight_movement_bytes=352321536 total_nvlink_bytes=2230495840 weight_to_token_ratio=0.187587 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228842144e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=936067072 dispatch_meta_bytes=915664 combine_payload_bytes=936067072 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=936982736 ingress_bytes=1288388608 link_bytes=1288388608 token_bytes=1873049808 weight_movement_bytes=352321536 total_nvlink_bytes=2225371344 weight_to_token_ratio=0.188100 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.214438405e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114684 dedup_ratio=0.500008 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 8134, 8190, 0, 0, 0, 8192, 8191, 0, 0, 0, 2, 0, 1, 0, 0, 17, 8177, 0, 1, 0, 0, 0, 8191, 0, 8192, 0, 0, 8192, 0, 8192, 2, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 103, 2, 21, 0, 0, 0, 8129, 0, 0, 0, 8191, 8190] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16268, 16386, 16390, 16382, 16388, 16622, 16254] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=932282368 dispatch_meta_bytes=867872 combine_payload_bytes=932282368 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=933150240 ingress_bytes=1196523520 link_bytes=1196523520 token_bytes=1865432608 weight_movement_bytes=264241152 total_nvlink_bytes=2129673760 weight_to_token_ratio=0.141651 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.700032259e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=915320 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940439416 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879963512 weight_movement_bytes=440401920 total_nvlink_bytes=2320365432 weight_to_token_ratio=0.234261 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917480 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425192 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932904 weight_movement_bytes=176160768 total_nvlink_bytes=2056093672 weight_to_token_ratio=0.093706 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114686 dedup_ratio=0.500000 router_tokens_per_expert=[8191, 8192, 8, 9, 7, 0, 8192, 0, 0, 0, 0, 0, 8171, 8192, 0, 0, 0, 8192, 8192, 0, 1, 0, 0, 0, 0, 8, 0, 29, 8170, 0, 0, 0, 0, 2, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 40, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16430, 16342, 16386, 16414, 16388, 16384, 16436, 16292] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131012 payload_rows_remote=114628 dup_rows=131132 dup_rows_remote=114668 dedup_ratio=0.500229 router_tokens_per_expert=[8147, 8129, 0, 0, 0, 0, 8167, 25, 24, 30, 25, 9, 8145, 8159, 20, 0, 34, 8162, 8173, 29, 16, 0, 0, 1, 5, 33, 3, 4, 8073, 3, 391, 10, 45, 98, 8161, 1, 8166, 0, 19, 8154, 0, 8154, 0, 0, 20, 29, 27, 28, 8154, 19, 12, 1, 1, 0, 11, 5, 19, 0, 7772, 0, 1, 34, 8161, 8163] payload_rows_per_dst=[16344, 16384, 16384, 16374, 16384, 16384, 16374, 16384] dup_rows_per_dst=[16258, 16440, 16446, 16992, 16572, 16464, 16366, 15594] combine_rows_per_src=[16344, 16384, 16384, 16374, 16384, 16384, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113324 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 8192, 8192, 0, 6, 8, 0, 0, 0, 0, 0, 4, 8178, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16304, 16380, 16286, 15332, 16502, 16384, 16136, 17748] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114666 dedup_ratio=0.500000 router_tokens_per_expert=[8190, 8189, 5, 1, 0, 1, 8190, 0, 0, 10, 0, 1, 8155, 8191, 0, 0, 0, 8191, 8192, 1, 0, 0, 0, 0, 1, 0, 2, 38, 8185, 0, 0, 1, 0, 1, 8191, 0, 8191, 0, 1, 8191, 0, 8191, 0, 0, 1, 1, 0, 0, 8191, 0, 0, 1, 12, 0, 0, 0, 0, 2, 8171, 0, 0, 1, 8191, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16330, 16384, 16452, 16384, 16382, 16406, 16346] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114924 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 34, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8074, 8192, 0, 0, 4, 8192, 8192, 0, 11, 0, 0, 0, 0, 0, 0, 134, 8168, 0, 4, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 2, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 9, 0, 2, 0, 6, 1, 8129, 0, 0, 0, 8192, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16452, 16148, 16414, 16612, 16384, 16388, 16404, 16270] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114678 dup_rows=131084 dup_rows_remote=114574 dedup_ratio=0.500046 router_tokens_per_expert=[8187, 8185, 0, 0, 0, 2, 8184, 4, 7, 7, 5, 7, 8179, 8185, 3, 0, 0, 8185, 8187, 5, 0, 0, 3, 0, 2, 0, 0, 11, 8157, 7, 63, 0, 10, 1, 8186, 0, 8186, 0, 6, 8185, 0, 8180, 0, 0, 6, 4, 2, 9, 8188, 3, 3, 0, 3, 0, 30, 1, 0, 1, 8113, 4, 3, 4, 8185, 8184] payload_rows_per_dst=[16382, 16382, 16384, 16384, 16384, 16378, 16384, 16382] dup_rows_per_dst=[16510, 16804, 16376, 16466, 16076, 16304, 16324, 16224] combine_rows_per_src=[16382, 16382, 16384, 16384, 16384, 16378, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=917184 combine_payload_bytes=939032576 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939949760 ingress_bytes=1115193344 link_bytes=1115193344 token_bytes=1878982336 weight_movement_bytes=176160768 total_nvlink_bytes=2055143104 weight_to_token_ratio=0.093753 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244617222e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917496 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940441592 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965688 weight_movement_bytes=352321536 total_nvlink_bytes=2232287224 weight_to_token_ratio=0.187408 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=912048 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940436144 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879960240 weight_movement_bytes=352321536 total_nvlink_bytes=2232281776 weight_to_token_ratio=0.187409 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131020 payload_rows_remote=114636 dup_rows=131124 dup_rows_remote=114742 dedup_ratio=0.500198 router_tokens_per_expert=[5468, 0, 0, 0, 2197, 0, 8192, 0, 0, 0, 54, 0, 0, 0, 5504, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 4709, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 6364, 0, 0, 8166, 0, 190, 0, 0, 8192, 0, 0, 8192, 17, 0, 0, 99, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16332, 16384, 16384] dup_rows_per_dst=[15330, 11116, 16388, 17504, 16382, 21024, 16764, 16616] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16332, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917416 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441512 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965608 weight_movement_bytes=176160768 total_nvlink_bytes=2056126376 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918448 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940442544 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879966640 weight_movement_bytes=176160768 total_nvlink_bytes=2056127408 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114692 dedup_ratio=0.500015 router_tokens_per_expert=[8190, 0, 8191, 1, 0, 0, 0, 8189, 0, 0, 0, 0, 23, 0, 0, 8190, 30, 0, 0, 8183, 1, 8190, 0, 0, 8191, 0, 0, 0, 8191, 3, 8190, 0, 735, 0, 2, 0, 2, 0, 8185, 0, 8190, 1, 8166, 5, 0, 0, 2, 2, 1, 1, 6082, 8186, 0, 137, 8191, 1226, 8191, 0, 0, 0, 1, 2, 0, 1] payload_rows_per_dst=[16382, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16378, 16426, 16424, 16384, 17846, 16348, 14882, 16388] combine_rows_per_src=[16382, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917008 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940359184 ingress_bytes=1467924480 link_bytes=1467924480 token_bytes=1879801360 weight_movement_bytes=528482304 total_nvlink_bytes=2408283664 weight_to_token_ratio=0.281137 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219764346e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 3] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939098112 dispatch_meta_bytes=917512 combine_payload_bytes=939098112 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940015624 ingress_bytes=1115258880 link_bytes=1115258880 token_bytes=1879113736 weight_movement_bytes=176160768 total_nvlink_bytes=2055274504 weight_to_token_ratio=0.093747 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244984196e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917504 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940408832 ingress_bytes=1379893248 link_bytes=1379893248 token_bytes=1879900160 weight_movement_bytes=440401920 total_nvlink_bytes=2320302080 weight_to_token_ratio=0.234269 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726826193e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114610 dedup_ratio=0.500061 router_tokens_per_expert=[8134, 15, 8165, 0, 15, 0, 0, 8174, 0, 17, 0, 24, 5, 23, 41, 8165, 106, 27, 4, 8170, 0, 8166, 0, 10, 8163, 0, 0, 2, 8156, 0, 8162, 36, 394, 25, 0, 30, 26, 8, 8140, 41, 8172, 27, 8161, 0, 71, 0, 0, 0, 18, 26, 6929, 8163, 33, 0, 8157, 668, 8159, 15, 0, 11, 0, 0, 0, 18] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16376, 16376] dup_rows_per_dst=[16292, 16496, 16582, 16318, 17258, 16478, 15298, 16366] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16376, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130544 payload_rows_remote=114160 dup_rows=131600 dup_rows_remote=115180 dedup_ratio=0.502014 router_tokens_per_expert=[8192, 0, 8192, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 8192, 126, 0, 0, 8190, 0, 8192, 0, 0, 8191, 0, 0, 0, 8192, 0, 8192, 0, 767, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 8190, 0, 1, 2, 0, 0, 0, 0, 6667, 7651, 0, 60, 8192, 1120, 8190, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[16384, 16384, 15856, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16374, 16384, 14890, 16434, 2888, 16830, 31380, 16420] combine_rows_per_src=[16384, 16384, 15856, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114676 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 8192, 14, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 22, 8174, 0, 8192, 0, 0, 8190, 0, 0, 0, 8192, 0, 8192, 0, 459, 0, 2, 0, 0, 0, 8184, 0, 8192, 0, 8137, 0, 5, 0, 0, 34, 0, 0, 7931, 7950, 0, 63, 8192, 20, 8192, 0, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16326, 16384, 16396, 16380, 17290, 16352, 15544, 16400] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=111298 dedup_ratio=0.500008 router_tokens_per_expert=[8191, 1, 8190, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 8191, 5, 1, 1, 8192, 0, 8191, 0, 1, 8192, 0, 0, 0, 8191, 0, 8191, 1, 1362, 0, 1, 0, 0, 0, 8190, 1, 8191, 0, 8139, 2, 0, 2, 0, 0, 1, 1, 6694, 8124, 0, 35, 8191, 215, 8191, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 16382, 16398, 16384, 13086, 16284, 19776, 16382] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131020 payload_rows_remote=114636 dup_rows=131124 dup_rows_remote=114696 dedup_ratio=0.500198 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 22, 0, 0, 8192, 841, 0, 33, 8191, 0, 8192, 0, 0, 8171, 0, 0, 0, 8192, 0, 8192, 0, 1542, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 7228, 6697, 0, 10, 8192, 221, 8192, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16332, 16384] dup_rows_per_dst=[16298, 16428, 18130, 16342, 19464, 16092, 11980, 16390] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16332, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 6, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114658 dup_rows=131102 dup_rows_remote=114456 dedup_ratio=0.500114 router_tokens_per_expert=[8180, 0, 8187, 0, 0, 5, 3, 8187, 0, 0, 0, 0, 0, 5, 18, 8184, 1, 6, 0, 8187, 0, 8186, 6, 5, 8185, 4, 0, 3, 8186, 9, 8183, 7, 3486, 35, 0, 2, 0, 1, 8186, 6, 8187, 4, 8188, 0, 0, 0, 0, 0, 5, 6, 4535, 8017, 0, 0, 8184, 289, 8187, 4, 0, 0, 3, 4, 6, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16354, 16384] dup_rows_per_dst=[16646, 16192, 16330, 16446, 23416, 16374, 9350, 16348] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16354, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917128 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940310152 ingress_bytes=1115553792 link_bytes=1115553792 token_bytes=1879703176 weight_movement_bytes=176160768 total_nvlink_bytes=2055863944 weight_to_token_ratio=0.093717 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246635580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=935198720 dispatch_meta_bytes=917360 combine_payload_bytes=935198720 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=936116080 ingress_bytes=1199439872 link_bytes=1199439872 token_bytes=1871314800 weight_movement_bytes=264241152 total_nvlink_bytes=2135555952 weight_to_token_ratio=0.141206 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.716362612e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 2, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8181, 8192, 0, 0, 0, 8192, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 20, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16364, 16394, 16424, 16384, 16384, 16388, 16346] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917456 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=940441552 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965648 weight_movement_bytes=704643072 total_nvlink_bytes=2584608720 weight_to_token_ratio=0.374817 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=112860 dedup_ratio=0.500015 router_tokens_per_expert=[0, 109, 0, 7, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8190, 0, 13, 0, 0, 8190, 0, 0, 8192, 0, 7267, 8192, 0, 0, 8178, 0, 4, 926, 0, 8145, 0, 0, 8182, 0, 0, 0, 8192, 0, 1, 0, 10, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 8191, 0, 0, 8107, 0, 8192, 0, 2, 0, 1] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16616, 16406, 14534, 18216, 16290, 16386, 16408, 16220] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=903936 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940411648 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879919360 weight_movement_bytes=352321536 total_nvlink_bytes=2232240896 weight_to_token_ratio=0.187413 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939098112 dispatch_meta_bytes=917328 combine_payload_bytes=939098112 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940015440 ingress_bytes=1115258880 link_bytes=1115258880 token_bytes=1879113552 weight_movement_bytes=176160768 total_nvlink_bytes=2055274320 weight_to_token_ratio=0.093747 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244984196e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=916456 combine_payload_bytes=939278336 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940194792 ingress_bytes=1291599872 link_bytes=1291599872 token_bytes=1879473128 weight_movement_bytes=352321536 total_nvlink_bytes=2231794664 weight_to_token_ratio=0.187458 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232420143e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441600 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965696 weight_movement_bytes=528482304 total_nvlink_bytes=2408448000 weight_to_token_ratio=0.281113 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=910176 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940401504 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879892832 weight_movement_bytes=352321536 total_nvlink_bytes=2232214368 weight_to_token_ratio=0.187416 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131000 payload_rows_remote=114616 dup_rows=131144 dup_rows_remote=114532 dedup_ratio=0.500275 router_tokens_per_expert=[0, 0, 45, 110, 46, 8159, 0, 8131, 8091, 34, 0, 21, 0, 8163, 117, 19, 0, 22, 8134, 28, 1, 8155, 22, 7623, 8143, 6, 29, 8111, 1, 28, 549, 40, 8086, 3, 38, 8163, 0, 26, 2, 8148, 21, 12, 1, 49, 19, 0, 45, 40, 8144, 7, 0, 25, 51, 0, 8164, 0, 0, 8020, 0, 8154, 5, 10, 11, 0] payload_rows_per_dst=[16382, 16384, 16374, 16374, 16384, 16384, 16384, 16334] dup_rows_per_dst=[16600, 16506, 15286, 17440, 16236, 16612, 16398, 16066] combine_rows_per_src=[16382, 16384, 16374, 16374, 16384, 16384, 16384, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130334 payload_rows_remote=113950 dup_rows=131810 dup_rows_remote=114480 dedup_ratio=0.502815 router_tokens_per_expert=[0, 8, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 1, 7, 0, 0, 8187, 0, 0, 8192, 0, 6258, 8192, 0, 0, 8192, 0, 5, 1932, 0, 8185, 0, 0, 8188, 0, 0, 4, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8185, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[15656, 16384, 16374, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14214, 16362, 11690, 22444, 17044, 16536, 16190, 17330] combine_rows_per_src=[15656, 16384, 16374, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=10 unused_prefetch_slots=22 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=5 remote_demand_peak=3 slot_utilization=0.312500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 10, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114688 dup_rows=131096 dup_rows_remote=117338 dedup_ratio=0.500092 router_tokens_per_expert=[0, 39, 0, 0, 0, 8176, 0, 8192, 8169, 0, 13, 0, 0, 8192, 0, 49, 0, 0, 8180, 0, 0, 8192, 0, 6879, 8186, 0, 0, 8192, 0, 46, 1338, 0, 8098, 0, 0, 8148, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 1, 0, 0, 0, 0, 8192, 92, 0, 8102, 0, 8192, 11, 3, 0, 0] payload_rows_per_dst=[16384, 16384, 16360, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16430, 16462, 13758, 19140, 16206, 16298, 16570, 16232] combine_rows_per_src=[16384, 16384, 16360, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114684 dedup_ratio=0.500008 router_tokens_per_expert=[0, 29, 1, 4, 0, 8191, 0, 8191, 8191, 0, 0, 1, 0, 8191, 12, 2, 0, 1, 8189, 1, 0, 8191, 0, 7710, 8179, 1, 1, 8192, 0, 0, 481, 0, 8191, 1, 0, 8189, 0, 0, 0, 8191, 0, 0, 0, 10, 1, 0, 0, 0, 8191, 3, 1, 1, 0, 0, 8191, 0, 0, 8151, 0, 8191, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16448, 16410, 15420, 17324, 16382, 16398, 16390, 16302] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114586 dedup_ratio=0.500000 router_tokens_per_expert=[0, 316, 0, 11, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 1, 50, 0, 0, 8191, 0, 0, 8192, 0, 3756, 8192, 0, 0, 8192, 0, 2, 4339, 0, 7953, 0, 0, 8169, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8172, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17038, 16486, 13568, 19008, 15906, 16338, 16384, 16344] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938934272 dispatch_meta_bytes=916592 combine_payload_bytes=938934272 prefetch_bytes=440401920 reduce_bytes=0 egress_bytes=939850864 ingress_bytes=1379336192 link_bytes=1379336192 token_bytes=1878785136 weight_movement_bytes=440401920 total_nvlink_bytes=2319187056 weight_to_token_ratio=0.234408 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.723706912e-05 prefetch_bytes_per_gemm_flop=2.466066919e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=113028 dedup_ratio=0.500084 router_tokens_per_expert=[0, 557, 16, 0, 17, 8187, 9, 8157, 8181, 11, 0, 4, 0, 8183, 93, 0, 7, 4, 8184, 5, 10, 8179, 7, 7359, 8178, 0, 8, 8185, 2, 0, 216, 0, 8185, 0, 7, 8189, 0, 6, 0, 8182, 9, 2, 3, 0, 0, 2, 0, 9, 8181, 0, 0, 7, 0, 0, 8185, 2, 0, 8159, 0, 8183, 0, 0, 2, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16376, 16370] dup_rows_per_dst=[18066, 16014, 31158, 438, 16624, 16120, 16356, 16318] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16376, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114692 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8190, 8115, 0, 0, 0, 0, 0, 0, 8192, 1, 22, 2, 0, 8192, 8192, 8192, 54, 798, 8190, 8192, 0, 8190, 8165, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8192, 8189, 0, 0, 0, 4, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 8, 0, 0, 8192, 1, 0, 7978, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16230, 16434, 18034, 16384, 16380, 16392, 15248, 15974] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=111258 dedup_ratio=0.500038 router_tokens_per_expert=[8192, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 8192, 32, 0, 1, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 1720, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 5503, 8074, 0, 4, 8192, 1087, 8192, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16374, 16384] dup_rows_per_dst=[16378, 16384, 16450, 16384, 19824, 16314, 12962, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=933478400 dispatch_meta_bytes=913720 combine_payload_bytes=933478400 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=934392120 ingress_bytes=1109639168 link_bytes=1109639168 token_bytes=1867870520 weight_movement_bytes=176160768 total_nvlink_bytes=2044031288 weight_to_token_ratio=0.094311 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.213516155e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=928104 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940452200 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879976296 weight_movement_bytes=440401920 total_nvlink_bytes=2320378216 weight_to_token_ratio=0.234259 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917480 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940425192 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932904 weight_movement_bytes=264241152 total_nvlink_bytes=2144174056 weight_to_token_ratio=0.140559 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917096 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940441192 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965288 weight_movement_bytes=352321536 total_nvlink_bytes=2232286824 weight_to_token_ratio=0.187409 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=910776 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940254648 ingress_bytes=1379745792 link_bytes=1379745792 token_bytes=1879598520 weight_movement_bytes=440401920 total_nvlink_bytes=2320000440 weight_to_token_ratio=0.234306 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.023674 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726000501e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917504 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940408832 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900160 weight_movement_bytes=352321536 total_nvlink_bytes=2232221696 weight_to_token_ratio=0.187415 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=903744 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940345920 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879788096 weight_movement_bytes=176160768 total_nvlink_bytes=2055948864 weight_to_token_ratio=0.093713 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130944 payload_rows_remote=114560 dup_rows=131200 dup_rows_remote=114756 dedup_ratio=0.500488 router_tokens_per_expert=[0, 8160, 8144, 24, 22, 0, 25, 59, 0, 8127, 0, 34, 0, 25, 8091, 8139, 8147, 28, 4411, 8160, 8120, 28, 8156, 8146, 0, 4, 0, 1, 25, 1, 0, 84, 6, 31, 0, 0, 0, 8073, 8159, 25, 28, 7, 0, 55, 0, 97, 8088, 0, 21, 0, 0, 0, 3745, 0, 44, 75, 0, 31, 37, 8149, 39, 43, 8154, 4] payload_rows_per_dst=[16384, 16304, 16340, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16484, 16250, 18268, 16460, 16208, 16444, 14556, 16530] combine_rows_per_src=[16384, 16304, 16340, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128526 payload_rows_remote=112142 dup_rows=133618 dup_rows_remote=115684 dedup_ratio=0.509712 router_tokens_per_expert=[0, 8192, 8190, 0, 0, 0, 0, 0, 0, 8192, 2, 1, 10, 0, 8192, 8191, 8192, 0, 2722, 8192, 8190, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6889, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6767, 0] payload_rows_per_dst=[16384, 16384, 13838, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16342, 3434, 16382, 16602, 16382, 30162, 17934] combine_rows_per_src=[16384, 16384, 13838, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=111742 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 8117, 0, 0, 0, 0, 0, 0, 8192, 1, 5, 27, 0, 8192, 8192, 8192, 30, 1491, 8192, 8192, 0, 8192, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 54, 0, 0, 0, 8192, 4, 0, 0, 1, 0, 6764, 8, 0, 0, 0, 0, 0, 8192, 0, 0, 8122, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16234, 16450, 19330, 16384, 16384, 16500, 13546, 16244] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938475520 dispatch_meta_bytes=917264 combine_payload_bytes=938475520 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939392784 ingress_bytes=1114636288 link_bytes=1114636288 token_bytes=1877868304 weight_movement_bytes=176160768 total_nvlink_bytes=2054029072 weight_to_token_ratio=0.093809 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.241497941e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=103392 dedup_ratio=0.500023 router_tokens_per_expert=[0, 8191, 8177, 1, 0, 0, 0, 0, 2, 8191, 0, 12, 0, 0, 8191, 8191, 8188, 0, 531, 8191, 8191, 0, 8191, 8177, 0, 0, 0, 1, 0, 1, 7, 0, 0, 0, 0, 0, 0, 8189, 8189, 1, 1, 0, 4, 1, 0, 3, 8184, 2, 1, 0, 0, 0, 7688, 0, 1, 13, 0, 1, 1, 8191, 1, 1, 8166, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16354, 16408, 5128, 16398, 16378, 16388, 27686, 16338] combine_rows_per_src=[16384, 16384, 16382, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114322 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 7567, 0, 0, 0, 0, 0, 0, 8192, 0, 183, 0, 0, 8192, 8192, 8192, 78, 4052, 8192, 8192, 0, 8192, 8037, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7874, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15134, 16750, 24334, 16392, 16384, 16384, 9946, 15748] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=13 export_slots=13 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 4, 1, 0, 4, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114638 dup_rows=131122 dup_rows_remote=113222 dedup_ratio=0.500191 router_tokens_per_expert=[0, 8183, 8187, 7, 5, 0, 10, 7, 0, 8173, 0, 0, 0, 4, 8177, 8180, 8185, 1, 2761, 8187, 8182, 15, 8183, 8187, 2, 0, 7, 2, 7, 0, 9, 19, 6, 5, 0, 0, 0, 8173, 8184, 4, 3, 12, 0, 10, 0, 2, 8166, 5, 9, 0, 0, 0, 5582, 0, 1, 0, 0, 6, 7, 8182, 6, 8, 8021, 0] payload_rows_per_dst=[16384, 16368, 16376, 16384, 16382, 16364, 16384, 16380] dup_rows_per_dst=[17900, 16126, 20982, 16288, 16362, 16218, 11166, 16080] combine_rows_per_src=[16384, 16368, 16376, 16384, 16382, 16364, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114974 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 3, 1, 0, 5964, 0, 1, 0, 2324, 8192, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 1, 8190, 8191, 2, 8191, 1, 8151, 0, 0, 8088, 2, 1, 0, 0, 8192, 8190, 0, 8190, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 25, 0, 0, 0, 36, 0, 1, 0, 0, 8185, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[11934, 21034, 16384, 16102, 16380, 16364, 16498, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=918667264 dispatch_meta_bytes=911304 combine_payload_bytes=918667264 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=919578568 ingress_bytes=1094828032 link_bytes=1094828032 token_bytes=1838245832 weight_movement_bytes=176160768 total_nvlink_bytes=2014406600 weight_to_token_ratio=0.095831 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.130579975e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114690 dedup_ratio=0.500008 router_tokens_per_expert=[0, 3, 0, 0, 0, 8192, 0, 8192, 8185, 0, 0, 0, 0, 8192, 0, 28, 0, 0, 8191, 0, 0, 8192, 0, 7855, 8192, 0, 0, 8192, 0, 9, 335, 0, 8191, 0, 0, 8191, 0, 0, 1, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8161, 0, 8192, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16390, 16426, 15710, 17072, 16384, 16384, 16384, 16324] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=905720 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=1145044992 egress_bytes=940429816 ingress_bytes=2172649472 link_bytes=2172649472 token_bytes=1879953912 weight_movement_bytes=1233125376 total_nvlink_bytes=3113079288 weight_to_token_ratio=0.655934 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.024621 gemm_active_groups=14 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216593013e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=872296 combine_payload_bytes=939474944 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940347240 ingress_bytes=1291796480 link_bytes=1291796480 token_bytes=1879822184 weight_movement_bytes=352321536 total_nvlink_bytes=2232143720 weight_to_token_ratio=0.187423 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233521066e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916040 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940440136 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879964232 weight_movement_bytes=528482304 total_nvlink_bytes=2408446536 weight_to_token_ratio=0.281113 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=911440 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940025936 ingress_bytes=1115275264 link_bytes=1115275264 token_bytes=1879140432 weight_movement_bytes=176160768 total_nvlink_bytes=2055301200 weight_to_token_ratio=0.093745 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245075940e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 6, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 5, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=918632 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940409960 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879901288 weight_movement_bytes=528482304 total_nvlink_bytes=2408383592 weight_to_token_ratio=0.281122 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131018 payload_rows_remote=114634 dup_rows=131126 dup_rows_remote=115174 dedup_ratio=0.500206 router_tokens_per_expert=[17, 11, 54, 0, 0, 7720, 2, 0, 6, 1161, 8143, 0, 13, 0, 38, 0, 31, 8151, 1, 35, 0, 8120, 8135, 0, 8160, 0, 7259, 22, 29, 8111, 0, 34, 27, 34, 8149, 8163, 13, 8151, 0, 10, 7872, 31, 18, 0, 19, 64, 0, 8164, 19, 56, 116, 3, 25, 0, 290, 35, 81, 30, 31, 8153, 8153, 8069, 17, 26] payload_rows_per_dst=[16348, 16384, 16380, 16384, 16370, 16384, 16384, 16384] dup_rows_per_dst=[15586, 18214, 16232, 14970, 16398, 15952, 17338, 16436] combine_rows_per_src=[16348, 16384, 16380, 16384, 16370, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=264241152 reduce_bytes=440401920 egress_bytes=940425216 ingress_bytes=1644150784 link_bytes=1644150784 token_bytes=1879932928 weight_movement_bytes=704643072 total_nvlink_bytes=2584576000 weight_to_token_ratio=0.374824 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206558088e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939081728 dispatch_meta_bytes=919232 combine_payload_bytes=939081728 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940000960 ingress_bytes=1115242496 link_bytes=1115242496 token_bytes=1879082688 weight_movement_bytes=176160768 total_nvlink_bytes=2055243456 weight_to_token_ratio=0.093748 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244892452e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130154 payload_rows_remote=113770 dup_rows=131990 dup_rows_remote=105948 dedup_ratio=0.503502 router_tokens_per_expert=[0, 0, 0, 0, 2, 6640, 0, 0, 0, 2281, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8144, 6, 8192, 0, 6899, 0, 0, 8157, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 8186, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15466, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[13284, 4562, 13408, 24824, 18798, 17302, 13770, 26042] combine_rows_per_src=[16384, 16384, 15466, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114220 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 35, 0, 3, 3885, 0, 0, 0, 2318, 8192, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 8192, 8162, 85, 8192, 0, 8064, 0, 0, 8106, 0, 2, 0, 0, 8192, 8192, 0, 8192, 0, 9, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 2174, 0, 1, 0, 0, 8114, 8191, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[7872, 21020, 16856, 15608, 16402, 16384, 20732, 16202] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114548 dedup_ratio=0.500008 router_tokens_per_expert=[2, 0, 0, 0, 6, 5858, 1, 0, 1, 2398, 8190, 0, 0, 0, 1, 1, 0, 8190, 2, 0, 0, 8191, 8190, 0, 8191, 0, 8064, 1, 1, 8170, 0, 8, 0, 1, 8191, 8191, 1, 8191, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 8191, 0, 1, 15, 1, 1, 0, 54, 1, 0, 0, 2, 8191, 8191, 8191, 0, 1] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[11732, 7998, 16382, 24578, 21092, 16380, 16526, 16386] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114674 dup_rows=131088 dup_rows_remote=103364 dedup_ratio=0.500061 router_tokens_per_expert=[19, 2, 40, 8, 0, 5614, 0, 0, 12, 2583, 8181, 0, 0, 0, 7, 0, 0, 8180, 0, 3, 0, 8174, 8186, 0, 8185, 0, 7795, 7, 6, 8188, 0, 65, 7, 8, 8182, 8186, 4, 8184, 0, 0, 7943, 11, 4, 0, 0, 5, 5, 8185, 0, 10, 61, 0, 3, 0, 436, 9, 0, 9, 3, 8185, 8178, 8179, 14, 6] payload_rows_per_dst=[16382, 16384, 16380, 16382, 16380, 16384, 16384, 16380] dup_rows_per_dst=[27724, 21552, 20290, 13020, 13942, 15926, 11652, 6982] combine_rows_per_src=[16382, 16384, 16380, 16382, 16380, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=112176 dedup_ratio=0.500107 router_tokens_per_expert=[0, 0, 64, 0, 0, 7455, 0, 0, 0, 1270, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8179, 0, 8192, 0, 7025, 0, 0, 8046, 0, 45, 0, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 8118, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16358, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[27922, 18924, 16096, 5946, 11692, 16384, 17866, 16270] combine_rows_per_src=[16384, 16384, 16358, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[1, 0, 8190, 0, 0, 5828, 0, 44, 0, 0, 0, 8191, 8192, 0, 8192, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 8192, 0, 8192, 0, 0, 2, 7637, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 634, 8192, 0, 8192, 0, 8186, 8187, 0, 0, 1, 8192, 4, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[11742, 20874, 16382, 16384, 16392, 15274, 17652, 16372] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=932003840 dispatch_meta_bytes=878872 combine_payload_bytes=932003840 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=932882712 ingress_bytes=1460486144 link_bytes=1460486144 token_bytes=1864886552 weight_movement_bytes=528482304 total_nvlink_bytes=2393368856 weight_to_token_ratio=0.283386 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.178112769e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 8180, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 8192, 8192, 8192, 0, 3894, 8192, 8192, 0, 8192, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4380, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 8114, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16360, 16408, 24162, 16384, 16384, 16384, 8762, 16228] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 6, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=916936 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940424648 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932360 weight_movement_bytes=352321536 total_nvlink_bytes=2232253896 weight_to_token_ratio=0.187412 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=915624 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940423336 ingress_bytes=1467990016 link_bytes=1467990016 token_bytes=1879931048 weight_movement_bytes=528482304 total_nvlink_bytes=2408413352 weight_to_token_ratio=0.281118 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220131320e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=872152 combine_payload_bytes=939409408 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940281560 ingress_bytes=1291730944 link_bytes=1291730944 token_bytes=1879690968 weight_movement_bytes=352321536 total_nvlink_bytes=2232012504 weight_to_token_ratio=0.187436 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233154091e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=907344 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940202064 ingress_bytes=1115455488 link_bytes=1115455488 token_bytes=1879496784 weight_movement_bytes=176160768 total_nvlink_bytes=2055657552 weight_to_token_ratio=0.093728 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246085119e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965696 weight_movement_bytes=264241152 total_nvlink_bytes=2144206848 weight_to_token_ratio=0.140556 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130990 payload_rows_remote=114608 dup_rows=131154 dup_rows_remote=115260 dedup_ratio=0.500313 router_tokens_per_expert=[0, 0, 8162, 0, 0, 4108, 44, 20, 0, 0, 43, 8156, 8145, 49, 8150, 3858, 15, 17, 0, 272, 32, 0, 1, 37, 35, 48, 0, 30, 0, 39, 13, 8137, 0, 0, 8148, 33, 8144, 22, 0, 0, 7798, 48, 42, 8158, 36, 0, 0, 56, 37, 0, 8135, 311, 8140, 13, 8155, 0, 8059, 8150, 2, 0, 0, 8150, 0, 24] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16362, 16382, 16358, 16352] dup_rows_per_dst=[8284, 7806, 16976, 16530, 16332, 15894, 16914, 32418] combine_rows_per_src=[16384, 16384, 16384, 16384, 16362, 16382, 16358, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.025568 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938868736 dispatch_meta_bytes=919472 combine_payload_bytes=938868736 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939788208 ingress_bytes=1115029504 link_bytes=1115029504 token_bytes=1878656944 weight_movement_bytes=176160768 total_nvlink_bytes=2054817712 weight_to_token_ratio=0.093770 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243699786e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=95196 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 8192, 0, 0, 2904, 0, 62, 0, 0, 0, 8192, 8192, 0, 8192, 5136, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 7354, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8189, 927, 8191, 0, 8192, 0, 8192, 8189, 0, 0, 0, 8192, 9, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[5932, 10272, 16386, 16208, 16300, 14706, 15392, 35878] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[2, 2, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=113870 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 8191, 0, 0, 7263, 1, 4, 0, 1, 1, 8191, 8191, 4, 8191, 692, 0, 1, 0, 1, 1, 1, 3, 0, 0, 1, 2, 0, 0, 0, 1, 8188, 0, 0, 8191, 0, 8191, 1, 43, 0, 7966, 1, 1, 8190, 1, 0, 0, 1, 0, 1, 8191, 411, 8191, 0, 8191, 0, 8189, 8191, 0, 0, 0, 8191, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[14534, 1394, 16394, 16382, 16468, 15938, 17204, 32760] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=118938 payload_rows_remote=102554 dup_rows=143206 dup_rows_remote=113716 dedup_ratio=0.546288 router_tokens_per_expert=[0, 2, 8184, 10, 0, 6514, 11, 36, 4, 0, 9, 8180, 8182, 26, 8183, 3968, 10, 5, 0, 12, 5, 4, 27, 9, 16, 10, 0, 0, 0, 1, 8, 8183, 0, 0, 8182, 7, 8170, 3, 0, 0, 5247, 10, 12, 8184, 8, 0, 0, 12, 0, 0, 8181, 440, 8185, 0, 8181, 0, 8176, 8180, 13, 0, 0, 8182, 94, 6] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16362, 4272] dup_rows_per_dst=[29490, 25068, 15776, 16418, 16398, 26914, 13140, 2] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16362, 4272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114698 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 8192, 0, 0, 6054, 0, 64, 0, 0, 0, 8182, 8192, 0, 8192, 4158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8192, 0, 8192, 0, 26, 0, 5714, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 382, 8192, 0, 8192, 0, 8192, 8188, 0, 0, 0, 8192, 6, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16384] dup_rows_per_dst=[21576, 24680, 16384, 16382, 16436, 11428, 17148, 7048] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130514 payload_rows_remote=114130 dup_rows=131630 dup_rows_remote=106896 dedup_ratio=0.502129 router_tokens_per_expert=[0, 0, 8190, 0, 0, 4007, 0, 226, 0, 0, 0, 8192, 8192, 0, 8192, 4175, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 7052, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 1020, 8192, 0, 8192, 0, 8174, 8108, 0, 0, 0, 8192, 7, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 15826] dup_rows_per_dst=[24794, 24734, 16386, 16384, 16384, 15960, 16986, 2] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 15826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114670 dup_rows=131092 dup_rows_remote=115384 dedup_ratio=0.500076 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 7398, 9, 8190, 0, 0, 0, 0, 0, 8192, 8192, 6154, 0, 0, 424, 0, 2689, 8192, 0, 7857, 0, 3, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 8188, 4, 0, 0, 0, 0, 0, 0, 8189, 1, 54, 2, 0, 0, 8192, 8186, 0, 8191, 0, 0, 0, 8192, 3, 8192] payload_rows_per_dst=[16384, 16384, 16366, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17208, 16254, 16284, 15708, 16376, 16382, 16490, 16390] combine_rows_per_src=[16384, 16384, 16366, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=839528 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940347240 ingress_bytes=1644150784 link_bytes=1644150784 token_bytes=1879854952 weight_movement_bytes=704643072 total_nvlink_bytes=2584498024 weight_to_token_ratio=0.374839 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206558088e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 3, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114686 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 10, 0, 0, 7518, 0, 0, 0, 692, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8188, 0, 8192, 0, 8029, 0, 0, 8171, 0, 6, 0, 0, 8192, 8192, 0, 8192, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 148, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15056, 17768, 16380, 16028, 16386, 16384, 16690, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=914224 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940421936 ingress_bytes=1556070400 link_bytes=1556070400 token_bytes=1879929648 weight_movement_bytes=616562688 total_nvlink_bytes=2496492336 weight_to_token_ratio=0.327971 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713344704e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=840122368 dispatch_meta_bytes=865080 combine_payload_bytes=840122368 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=840987448 ingress_bytes=1016283136 link_bytes=1016283136 token_bytes=1681109816 weight_movement_bytes=176160768 total_nvlink_bytes=1857270584 weight_to_token_ratio=0.104788 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.023674 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.690761344e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917504 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940359680 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879801856 weight_movement_bytes=352321536 total_nvlink_bytes=2232123392 weight_to_token_ratio=0.187425 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131014 payload_rows_remote=114630 dup_rows=131130 dup_rows_remote=114566 dedup_ratio=0.500221 router_tokens_per_expert=[45, 42, 26, 0, 46, 0, 3018, 0, 8129, 54, 37, 99, 0, 17, 8144, 8100, 5752, 0, 51, 4462, 26, 3111, 8149, 0, 8102, 0, 7, 0, 0, 30, 8135, 40, 0, 28, 0, 18, 42, 0, 1, 8132, 8135, 0, 36, 1, 63, 0, 0, 86, 8147, 0, 0, 0, 29, 0, 8151, 8156, 13, 8153, 0, 12, 57, 8125, 0, 8065] payload_rows_per_dst=[16384, 16384, 16384, 16370, 16384, 16384, 16360, 16364] dup_rows_per_dst=[6258, 16488, 25026, 17644, 16384, 16564, 16356, 16410] combine_rows_per_src=[16384, 16384, 16384, 16370, 16384, 16384, 16360, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934952960 dispatch_meta_bytes=884104 combine_payload_bytes=934952960 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=935837064 ingress_bytes=1463435264 link_bytes=1463435264 token_bytes=1870790024 weight_movement_bytes=528482304 total_nvlink_bytes=2399272328 weight_to_token_ratio=0.282492 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.194626610e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=920216 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940296856 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879673496 weight_movement_bytes=176160768 total_nvlink_bytes=2055834264 weight_to_token_ratio=0.093719 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917496 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441592 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965688 weight_movement_bytes=528482304 total_nvlink_bytes=2408447992 weight_to_token_ratio=0.281113 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939048960 dispatch_meta_bytes=916784 combine_payload_bytes=939048960 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=939965744 ingress_bytes=1203290112 link_bytes=1203290112 token_bytes=1879014704 weight_movement_bytes=264241152 total_nvlink_bytes=2143255856 weight_to_token_ratio=0.140628 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.737922349e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131016 payload_rows_remote=114632 dup_rows=131128 dup_rows_remote=106280 dedup_ratio=0.500214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3801, 2, 8192, 0, 0, 0, 0, 0, 8192, 8192, 5976, 0, 0, 2515, 0, 4638, 8192, 0, 7561, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 85, 0, 0, 0, 8192, 8190, 0, 8192, 0, 0, 0, 8192, 1, 8192] payload_rows_per_dst=[16384, 16384, 16330, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[7606, 16384, 16164, 17078, 16384, 16130, 16534, 24848] combine_rows_per_src=[16384, 16384, 16330, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114692 dedup_ratio=0.500008 router_tokens_per_expert=[1, 1, 1, 0, 0, 1, 7343, 47, 8191, 1, 0, 0, 0, 1, 8190, 8190, 1772, 0, 1, 3655, 1, 3947, 8191, 2, 7807, 0, 0, 0, 0, 1, 8190, 1, 0, 1, 0, 1, 1, 0, 0, 8191, 8189, 0, 0, 1, 1, 0, 0, 6, 8191, 0, 0, 0, 1, 0, 8191, 8191, 0, 8191, 0, 0, 0, 8190, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[14786, 16382, 10564, 15614, 16384, 16392, 16382, 24570] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131032 payload_rows_remote=114648 dup_rows=131112 dup_rows_remote=98860 dedup_ratio=0.500153 router_tokens_per_expert=[8, 9, 6, 0, 9, 0, 7919, 5, 8180, 12, 2, 18, 2, 5, 8182, 8178, 7305, 0, 11, 1022, 0, 241, 8180, 0, 8028, 8, 0, 1, 0, 10, 8169, 8, 0, 3, 0, 7, 11, 0, 0, 8174, 8181, 0, 7, 0, 10, 0, 0, 42, 8182, 0, 13, 0, 8, 9, 8183, 8185, 2, 8184, 0, 4, 12, 8179, 0, 8158] payload_rows_per_dst=[16384, 16384, 16366, 16368, 16384, 16384, 16380, 16382] dup_rows_per_dst=[32252, 16434, 8972, 16080, 16400, 16524, 16314, 8136] combine_rows_per_src=[16384, 16384, 16366, 16368, 16384, 16384, 16380, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=117398 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 5305, 271, 8190, 0, 0, 0, 0, 0, 8183, 8192, 2201, 1, 0, 3920, 0, 4811, 8192, 0, 8147, 0, 26, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8157, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 2, 0, 0, 0, 8192, 8129, 0, 8192, 0, 0, 0, 8192, 15, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[19606, 16362, 13674, 16316, 16382, 16314, 16266, 16152] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939065344 dispatch_meta_bytes=883648 combine_payload_bytes=939065344 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=939948992 ingress_bytes=1379467264 link_bytes=1379467264 token_bytes=1879014336 weight_movement_bytes=440401920 total_nvlink_bytes=2319416256 weight_to_token_ratio=0.234379 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.024621 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.724440860e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2464, 1, 8192, 0, 0, 0, 0, 0, 8192, 8192, 5819, 0, 0, 2355, 0, 5787, 8192, 0, 8145, 0, 30, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8191, 46, 0, 0, 0, 0, 8192, 8131, 0, 8192, 0, 0, 0, 8192, 6, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[21314, 16384, 19730, 16320, 16384, 16384, 16352, 8204] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114976 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 7637, 2, 2, 0, 8192, 0, 8168, 8191, 0, 0, 0, 3, 8179, 0, 0, 8192, 1, 709, 0, 0, 8190, 0, 0, 0, 2, 8190, 2, 8192, 8044, 0, 8190, 2, 0, 8192, 7, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 18, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15282, 16344, 17800, 16096, 16398, 16382, 16366, 16404] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 7349, 0, 11, 0, 0, 0, 8192, 8192, 0, 8192, 1264, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 7487, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8184, 158, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8192, 121, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14720, 16244, 16388, 16384, 16384, 14974, 16684, 19294] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130880 payload_rows_remote=114496 dup_rows=131264 dup_rows_remote=114606 dedup_ratio=0.500732 router_tokens_per_expert=[29, 18, 21, 6863, 0, 0, 0, 8118, 45, 8114, 8060, 41, 0, 3, 0, 8167, 0, 0, 8135, 0, 3324, 0, 0, 8143, 22, 0, 52, 0, 8157, 10, 8141, 6034, 0, 8148, 0, 58, 8135, 0, 44, 0, 27, 43, 49, 0, 8131, 74, 43, 8154, 39, 155, 0, 0, 0, 8147, 0, 51, 8146, 3, 28, 0, 38, 0, 27, 35] payload_rows_per_dst=[16336, 16320, 16304, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[13762, 16208, 22900, 12134, 16386, 16658, 16710, 16506] combine_rows_per_src=[16336, 16320, 16304, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917512 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940425224 ingress_bytes=1556070400 link_bytes=1556070400 token_bytes=1879932936 weight_movement_bytes=616562688 total_nvlink_bytes=2496495624 weight_to_token_ratio=0.327971 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713344704e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=854032 combine_payload_bytes=939196416 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940050448 ingress_bytes=1291517952 link_bytes=1291517952 token_bytes=1879246864 weight_movement_bytes=352321536 total_nvlink_bytes=2231568400 weight_to_token_ratio=0.187480 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231961425e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=928344 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940452440 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879976536 weight_movement_bytes=616562688 total_nvlink_bytes=2496539224 weight_to_token_ratio=0.327963 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=23808 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940441600 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965696 weight_movement_bytes=440401920 total_nvlink_bytes=2320367616 weight_to_token_ratio=0.234261 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918656 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940442752 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879966848 weight_movement_bytes=352321536 total_nvlink_bytes=2232288384 weight_to_token_ratio=0.187408 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937951232 dispatch_meta_bytes=916408 combine_payload_bytes=937951232 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938867640 ingress_bytes=1290272768 link_bytes=1290272768 token_bytes=1876818872 weight_movement_bytes=352321536 total_nvlink_bytes=2229140408 weight_to_token_ratio=0.187723 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224988914e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8125, 0, 0, 0, 8192, 0, 8190, 8191, 0, 2, 0, 0, 8188, 0, 0, 8192, 0, 245, 0, 0, 8192, 0, 0, 0, 0, 8192, 5, 8192, 8011, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 0, 0, 1, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16250, 16304, 9722, 23184, 16292, 16480, 16456, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114702 dedup_ratio=0.500008 router_tokens_per_expert=[1, 1, 0, 8132, 0, 0, 0, 8191, 1, 8190, 8173, 1, 0, 1, 0, 8191, 0, 0, 8191, 0, 763, 0, 1, 8191, 0, 1, 1, 0, 8185, 1, 8191, 7378, 0, 8191, 0, 1, 8191, 0, 1, 0, 0, 0, 0, 1, 8191, 1, 1, 8191, 1, 0, 0, 0, 0, 8191, 0, 1, 8191, 0, 1, 0, 0, 1, 131, 1] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16266, 16350, 17908, 14760, 16384, 16386, 16372, 16648] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131026 payload_rows_remote=114642 dup_rows=131118 dup_rows_remote=112670 dedup_ratio=0.500175 router_tokens_per_expert=[0, 0, 0, 7519, 0, 0, 0, 8192, 0, 8190, 8121, 0, 0, 0, 5, 8159, 0, 0, 8192, 0, 1032, 0, 0, 8192, 0, 0, 0, 0, 8155, 40, 8192, 7862, 0, 8192, 0, 0, 8190, 22, 0, 0, 0, 0, 0, 22, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 27, 0] payload_rows_per_dst=[16384, 16338, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15038, 16238, 18448, 15730, 16424, 16428, 16384, 16428] combine_rows_per_src=[16384, 16338, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=108054 dedup_ratio=0.500031 router_tokens_per_expert=[8, 5, 16, 7038, 0, 0, 0, 8176, 8, 8173, 8139, 9, 0, 1, 0, 8186, 0, 0, 8180, 0, 1506, 0, 4, 8180, 2, 3, 12, 0, 8184, 0, 8184, 7813, 8, 8184, 0, 11, 8183, 0, 12, 0, 4, 9, 0, 0, 8181, 19, 1, 8184, 8, 47, 0, 0, 0, 8180, 0, 8, 8182, 1, 8, 0, 5, 0, 1, 9] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[23026, 17758, 9844, 16996, 16402, 15926, 16074, 15054] combine_rows_per_src=[16384, 16384, 16376, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114688 dup_rows=131086 dup_rows_remote=115470 dedup_ratio=0.500053 router_tokens_per_expert=[0, 0, 0, 7572, 0, 0, 0, 8192, 0, 8165, 7893, 0, 0, 0, 0, 8100, 0, 0, 8192, 0, 735, 0, 0, 8192, 0, 0, 0, 0, 8173, 23, 8192, 8027, 0, 8192, 0, 0, 8192, 7, 0, 0, 0, 0, 0, 22, 8192, 0, 0, 8192, 0, 0, 0, 1, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 434, 0] payload_rows_per_dst=[16384, 16370, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15144, 15616, 17854, 16062, 16398, 16428, 16332, 17252] combine_rows_per_src=[16384, 16370, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=115230 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 5, 8192, 0, 2, 0, 0, 8190, 8192, 8191, 8192, 0, 59, 8191, 2, 7862, 1, 0, 0, 2, 0, 0, 2, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8189, 0, 0, 7860, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 731, 2, 0, 8057, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[16382, 16382, 16398, 15846, 16392, 16378, 15718, 17580] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 1, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114694 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6021, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 5234, 0, 0, 2790, 0, 2350, 8192, 0, 8172, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 8192, 8189, 0, 8192, 0, 0, 0, 8192, 1, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[12042, 16384, 20748, 16344, 16378, 16382, 16408, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130820 payload_rows_remote=114436 dup_rows=131324 dup_rows_remote=114906 dedup_ratio=0.500961 router_tokens_per_expert=[42, 58, 14, 34, 32, 380, 0, 0, 63, 0, 1, 4, 0, 12, 0, 8143, 0, 7574, 54, 0, 0, 17, 8127, 8122, 8157, 8149, 41, 182, 7893, 0, 7956, 0, 1, 0, 0, 80, 0, 1040, 8107, 8144, 39, 0, 33, 0, 8108, 41, 8153, 27, 39, 4948, 7791, 44, 20, 49, 174, 8066, 13, 0, 3, 2986, 0, 0, 8072, 39] payload_rows_per_dst=[16384, 16370, 16364, 16304, 16376, 16384, 16256, 16382] dup_rows_per_dst=[17348, 16330, 15170, 15840, 18368, 16418, 9874, 21976] combine_rows_per_src=[16384, 16370, 16364, 16304, 16376, 16384, 16256, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[4, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917552 combine_payload_bytes=939507712 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940425264 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932976 weight_movement_bytes=264241152 total_nvlink_bytes=2144174128 weight_to_token_ratio=0.140559 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=909248 combine_payload_bytes=939147264 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940056512 ingress_bytes=1555709952 link_bytes=1555709952 token_bytes=1879203776 weight_movement_bytes=616562688 total_nvlink_bytes=2495766464 weight_to_token_ratio=0.328098 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.711326346e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=890936 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940349496 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879808056 weight_movement_bytes=176160768 total_nvlink_bytes=2055968824 weight_to_token_ratio=0.093712 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=920632 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940444728 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879968824 weight_movement_bytes=440401920 total_nvlink_bytes=2320370744 weight_to_token_ratio=0.234260 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.017992 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=919656 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940410984 ingress_bytes=1556054016 link_bytes=1556054016 token_bytes=1879902312 weight_movement_bytes=616562688 total_nvlink_bytes=2496465000 weight_to_token_ratio=0.327976 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713252961e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917528 combine_payload_bytes=939524096 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940441624 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965720 weight_movement_bytes=264241152 total_nvlink_bytes=2144206872 weight_to_token_ratio=0.140556 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=112356 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8189, 0, 0, 0, 0, 8192, 8125, 8192, 8192, 0, 20, 8125, 3, 7925, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 7415, 8183, 0, 0, 0, 0, 8192, 0, 0, 0, 1166, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15232, 16384, 16244, 16698, 16090, 16384, 15324, 18716] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937459712 dispatch_meta_bytes=917368 combine_payload_bytes=937459712 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=938377080 ingress_bytes=1113620480 link_bytes=1113620480 token_bytes=1875836792 weight_movement_bytes=176160768 total_nvlink_bytes=2051997560 weight_to_token_ratio=0.093910 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.235809840e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29991671920 prefetch_bytes=3082813440 reduce_bytes=2202009600 weight_movement_bytes=5284823040 egress_bytes=15003146352 ingress_bytes=20273348608 link_bytes=20273348608 total_nvlink_bytes=35276494960 weight_to_token_ratio=0.176210 prefetch_share_of_ingress=0.152062 dedup_ratio=0.501323 prefetch_slots=70 unused_prefetch_slots=442 slot_utilization=0.136719 reduce_slots=25 remote_demand=35 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021129 gemm_inner_pad_upper_bound=34036 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.095143106e-05 prefetch_bytes_per_gemm_flop=1.078904277e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=908176 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940432272 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879956368 weight_movement_bytes=176160768 total_nvlink_bytes=2056117136 weight_to_token_ratio=0.093705 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29983009784 prefetch_bytes=3082813440 reduce_bytes=1321205760 weight_movement_bytes=4404019200 egress_bytes=14998825976 ingress_bytes=19388203008 link_bytes=19388203008 total_nvlink_bytes=34387028984 weight_to_token_ratio=0.146884 prefetch_share_of_ingress=0.159005 dedup_ratio=0.501586 prefetch_slots=70 unused_prefetch_slots=442 slot_utilization=0.136719 reduce_slots=15 remote_demand=35 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021129 gemm_inner_pad_upper_bound=33909 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.785365238e-05 prefetch_bytes_per_gemm_flop=1.078904277e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114682 dup_rows=131092 dup_rows_remote=121938 dedup_ratio=0.500076 router_tokens_per_expert=[0, 1, 0, 2, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8190, 0, 0, 0, 1, 8191, 8062, 8190, 8190, 0, 165, 8187, 0, 7996, 0, 0, 0, 0, 2, 0, 4, 8189, 8191, 1, 0, 1, 0, 8191, 1, 8191, 1, 1, 4578, 8180, 1, 1, 1, 0, 8191, 0, 3, 1, 3773, 0, 1, 8191, 1] payload_rows_per_dst=[16384, 16382, 16382, 16382, 16384, 16384, 16370, 16384] dup_rows_per_dst=[16404, 16382, 16124, 16314, 16388, 16388, 9154, 23938] combine_rows_per_src=[16384, 16382, 16382, 16382, 16384, 16384, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114814 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 8175, 0, 0, 0, 0, 8192, 8143, 8192, 8192, 0, 140, 8166, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 6075, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 2183, 0, 0, 8109, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16258, 16548, 16384, 16384, 15752, 16978] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131032 payload_rows_remote=114648 dup_rows=131112 dup_rows_remote=114558 dedup_ratio=0.500153 router_tokens_per_expert=[4, 14, 3, 13, 5, 63, 0, 0, 17, 0, 0, 2, 0, 2, 0, 8183, 0, 8156, 4, 0, 0, 0, 8176, 8177, 8184, 8183, 20, 131, 8057, 0, 8074, 0, 0, 0, 0, 13, 0, 12, 8173, 8182, 9, 4, 7, 0, 8178, 7, 8181, 9, 9, 7405, 7580, 8, 5, 11, 46, 8149, 0, 0, 1, 1423, 0, 0, 8184, 8] payload_rows_per_dst=[16384, 16384, 16372, 16374, 16372, 16378, 16384, 16384] dup_rows_per_dst=[16554, 16458, 20796, 19516, 14606, 16358, 23976, 2848] combine_rows_per_src=[16384, 16384, 16372, 16374, 16372, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114696 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8188, 0, 0, 0, 0, 8192, 6928, 8192, 8192, 0, 54, 8189, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8046, 8121, 0, 0, 0, 0, 8192, 0, 0, 0, 1445, 0, 0, 8182, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16376] dup_rows_per_dst=[16384, 16384, 13848, 16484, 16384, 16384, 27058, 8154] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8176, 0, 0, 0, 8192, 0, 8191, 8190, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 150, 0, 0, 8192, 0, 0, 0, 0, 8186, 0, 8192, 8059, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16352, 16378, 16684, 16112, 16388, 16384, 16378, 16396] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29887125888 prefetch_bytes=2378170368 reduce_bytes=2818572288 weight_movement_bytes=5196742656 egress_bytes=14950766976 ingress_bytes=20133101568 link_bytes=20133101568 total_nvlink_bytes=35083868544 weight_to_token_ratio=0.173879 prefetch_share_of_ingress=0.118122 dedup_ratio=0.502818 prefetch_slots=54 unused_prefetch_slots=458 slot_utilization=0.105469 reduce_slots=32 remote_demand=27 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020952 gemm_inner_pad_upper_bound=34163 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.046060301e-05 prefetch_bytes_per_gemm_flop=8.322975852e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=946480 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940421424 ingress_bytes=1467957248 link_bytes=1467957248 token_bytes=1879896368 weight_movement_bytes=528482304 total_nvlink_bytes=2408378672 weight_to_token_ratio=0.281123 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219947833e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918008 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940442104 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879966200 weight_movement_bytes=528482304 total_nvlink_bytes=2408448504 weight_to_token_ratio=0.281113 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=916824 combine_payload_bytes=939196416 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940113240 ingress_bytes=1291517952 link_bytes=1291517952 token_bytes=1879309656 weight_movement_bytes=352321536 total_nvlink_bytes=2231631192 weight_to_token_ratio=0.187474 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231961425e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917504 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940376064 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879834624 weight_movement_bytes=352321536 total_nvlink_bytes=2232156160 weight_to_token_ratio=0.187422 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=264241152 egress_bytes=940441584 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965680 weight_movement_bytes=264241152 total_nvlink_bytes=2144206832 weight_to_token_ratio=0.140556 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29990968368 prefetch_bytes=3082813440 reduce_bytes=2642411520 weight_movement_bytes=5725224960 egress_bytes=15002803248 ingress_bytes=20713390080 link_bytes=20713390080 total_nvlink_bytes=35716193328 weight_to_token_ratio=0.190898 prefetch_share_of_ingress=0.148832 dedup_ratio=0.501413 prefetch_slots=70 unused_prefetch_slots=442 slot_utilization=0.136719 reduce_slots=30 remote_demand=35 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020952 gemm_inner_pad_upper_bound=34544 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.249146141e-05 prefetch_bytes_per_gemm_flop=1.078904277e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29706673600 prefetch_bytes=3611295744 reduce_bytes=1497366528 weight_movement_bytes=5108662272 egress_bytes=14860524992 ingress_bytes=19954810880 link_bytes=19954810880 total_nvlink_bytes=34815335872 weight_to_token_ratio=0.171970 prefetch_share_of_ingress=0.180974 dedup_ratio=0.505452 prefetch_slots=82 unused_prefetch_slots=430 slot_utilization=0.160156 reduce_slots=17 remote_demand=41 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020774 gemm_inner_pad_upper_bound=35179 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.983663211e-05 prefetch_bytes_per_gemm_flop=1.263859296e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29978486848 prefetch_bytes=2642411520 reduce_bytes=5284823040 weight_movement_bytes=7927234560 egress_bytes=14996580416 ingress_bytes=22909140992 link_bytes=22909140992 total_nvlink_bytes=37905721408 weight_to_token_ratio=0.264431 prefetch_share_of_ingress=0.115343 dedup_ratio=0.501623 prefetch_slots=60 unused_prefetch_slots=452 slot_utilization=0.117188 reduce_slots=60 remote_demand=30 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021070 gemm_inner_pad_upper_bound=33909 gemm_flops_step=285735584268288 bytes_per_gemm_flop=8.017601676e-05 prefetch_bytes_per_gemm_flop=9.247750947e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29955421528 prefetch_bytes=2642411520 reduce_bytes=3875536896 weight_movement_bytes=6517948416 egress_bytes=14984967512 ingress_bytes=21488402432 link_bytes=21488402432 total_nvlink_bytes=36473369944 weight_to_token_ratio=0.217588 prefetch_share_of_ingress=0.122969 dedup_ratio=0.501886 prefetch_slots=60 unused_prefetch_slots=452 slot_utilization=0.117188 reduce_slots=44 remote_demand=30 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021484 gemm_inner_pad_upper_bound=32766 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.520380245e-05 prefetch_bytes_per_gemm_flop=9.247750947e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114594 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8187, 0, 0, 0, 0, 8192, 8139, 8192, 8192, 0, 91, 8192, 0, 6868, 0, 0, 0, 0, 0, 0, 48, 8192, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 7282, 8091, 0, 0, 0, 0, 8192, 0, 0, 0, 2268, 0, 0, 8178, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16270, 13918, 16480, 16384, 14362, 20892] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917120 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940424832 ingress_bytes=1379909632 link_bytes=1379909632 token_bytes=1879932544 weight_movement_bytes=440401920 total_nvlink_bytes=2320334464 weight_to_token_ratio=0.234265 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726917937e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=30005973904 prefetch_bytes=2466250752 reduce_bytes=3347054592 weight_movement_bytes=5813305344 egress_bytes=15010304912 ingress_bytes=20808974336 link_bytes=20808974336 total_nvlink_bytes=35819279248 weight_to_token_ratio=0.193738 prefetch_share_of_ingress=0.118519 dedup_ratio=0.501247 prefetch_slots=56 unused_prefetch_slots=456 slot_utilization=0.109375 reduce_slots=38 remote_demand=28 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021544 gemm_inner_pad_upper_bound=32512 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.282598137e-05 prefetch_bytes_per_gemm_flop=8.631234217e-06 anomalies=none - [2026-09-16 21:39:31] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 909.5 | throughput per GPU (TFLOP/s/GPU): 366.6 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.038128E+01 | loss scale: 1.0 | grad norm: 16.230 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122672 payload_rows_remote=107430 dup_rows=139472 dup_rows_remote=121798 dedup_ratio=0.532043 router_tokens_per_expert=[21, 68, 2413, 7, 641, 4, 3101, 7, 5920, 109, 235, 5078, 2352, 1031, 0, 6572, 2190, 2706, 2033, 2355, 5758, 3336, 562, 3276, 13, 5664, 164, 1998, 985, 32, 395, 2670, 6614, 19, 100, 3481, 5082, 26, 1482, 3897, 4455, 1924, 1649, 1321, 1, 1130, 3835, 4002, 592, 7048, 1275, 6, 5, 15, 1974, 950, 0, 7127, 4744, 887, 784, 2355, 2537, 59] payload_rows_per_dst=[15820, 15642, 15242, 15872, 13662, 15950, 16256, 14228] dup_rows_per_dst=[20096, 20626, 17674, 20248, 9912, 20684, 21728, 8504] combine_rows_per_src=[15820, 15642, 15242, 15872, 13662, 15950, 16256, 14228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124060 payload_rows_remote=108298 dup_rows=138084 dup_rows_remote=127666 dedup_ratio=0.526749 router_tokens_per_expert=[20, 32, 1782, 0, 525, 0, 3125, 0, 5710, 126, 218, 4522, 2862, 1299, 0, 6574, 1657, 2594, 1402, 1664, 6567, 3841, 465, 3133, 0, 5643, 87, 2144, 885, 23, 465, 2332, 7336, 75, 115, 3927, 5655, 0, 1702, 4585, 3892, 1639, 1169, 952, 5, 1453, 3128, 3570, 202, 7406, 2134, 0, 2, 25, 2322, 999, 0, 7452, 4761, 1155, 772, 2896, 2018, 28] payload_rows_per_dst=[15576, 15010, 14736, 15990, 16266, 14470, 15762, 16250] dup_rows_per_dst=[10064, 14464, 13294, 20600, 28702, 14056, 10418, 26486] combine_rows_per_src=[15576, 15010, 14736, 15990, 16266, 14470, 15762, 16250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125694 payload_rows_remote=109956 dup_rows=136450 dup_rows_remote=118164 dedup_ratio=0.520515 router_tokens_per_expert=[16, 7, 2067, 1, 606, 0, 3141, 0, 4591, 142, 181, 5220, 1753, 1534, 0, 6669, 2590, 2738, 1925, 1368, 6168, 4129, 353, 3357, 0, 5956, 20, 2241, 811, 6, 175, 1966, 7331, 38, 122, 3404, 4914, 8, 1042, 4839, 3820, 1306, 1080, 718, 0, 1065, 3603, 5420, 236, 7584, 1850, 0, 0, 2, 2463, 1401, 1, 7208, 4974, 1260, 584, 2969, 2081, 18] payload_rows_per_dst=[15694, 14804, 15146, 15920, 16158, 15738, 16046, 16188] dup_rows_per_dst=[13324, 12038, 17774, 19508, 22492, 18286, 11026, 22002] combine_rows_per_src=[15694, 14804, 15146, 15920, 16158, 15738, 16046, 16188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121742 payload_rows_remote=105542 dup_rows=140402 dup_rows_remote=114982 dedup_ratio=0.535591 router_tokens_per_expert=[25, 53, 2382, 4, 492, 3, 2622, 22, 5694, 122, 187, 4982, 2143, 1136, 6, 6540, 2764, 2636, 2302, 1542, 6373, 2932, 366, 3157, 5, 5168, 51, 2336, 1442, 25, 321, 2322, 7034, 24, 123, 3150, 6509, 6, 1065, 4973, 3924, 1644, 1071, 1265, 6, 1105, 3483, 4485, 252, 7047, 1597, 0, 25, 19, 2863, 1077, 0, 7475, 4440, 844, 842, 2461, 2099, 9] payload_rows_per_dst=[16264, 16200, 15828, 14436, 13258, 15382, 16334, 14040] dup_rows_per_dst=[22028, 25420, 23098, 13114, 6450, 18566, 24376, 7350] combine_rows_per_src=[16264, 16200, 15828, 14436, 13258, 15382, 16334, 14040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122858 payload_rows_remote=106502 dup_rows=139286 dup_rows_remote=105812 dedup_ratio=0.531334 router_tokens_per_expert=[17, 44, 2002, 8, 547, 1, 2358, 10, 5428, 118, 117, 4715, 2672, 797, 6, 6449, 2330, 3013, 2065, 1367, 6588, 4084, 355, 3021, 1, 4561, 28, 1796, 973, 25, 369, 2458, 7351, 54, 130, 3442, 6479, 14, 993, 4490, 4131, 1485, 916, 983, 6, 971, 3552, 5168, 160, 7103, 1547, 13, 1, 16, 2514, 1085, 1, 7609, 5207, 1407, 683, 2763, 2433, 42] payload_rows_per_dst=[15270, 14776, 13978, 15592, 16334, 15102, 15450, 16356] dup_rows_per_dst=[9406, 12508, 12088, 17276, 28498, 16608, 9428, 33474] combine_rows_per_src=[15270, 14776, 13978, 15592, 16334, 15102, 15450, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=880066560 dispatch_meta_bytes=916912 combine_payload_bytes=880066560 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=880983472 ingress_bytes=1408548864 link_bytes=1408548864 token_bytes=1761050032 weight_movement_bytes=528482304 total_nvlink_bytes=2289532336 weight_to_token_ratio=0.300095 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=12672 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.887285681e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=900759552 dispatch_meta_bytes=912480 combine_payload_bytes=900759552 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=901672032 ingress_bytes=1253081088 link_bytes=1253081088 token_bytes=1802431584 weight_movement_bytes=352321536 total_nvlink_bytes=2154753120 weight_to_token_ratio=0.195470 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=16000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.016731031e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=887177216 dispatch_meta_bytes=943856 combine_payload_bytes=887177216 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=888121072 ingress_bytes=1063337984 link_bytes=1063337984 token_bytes=1775298288 weight_movement_bytes=176160768 total_nvlink_bytes=1951459056 weight_to_token_ratio=0.099229 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.954248851e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124522 payload_rows_remote=108722 dup_rows=137622 dup_rows_remote=118024 dedup_ratio=0.524986 router_tokens_per_expert=[16, 68, 2432, 2, 478, 0, 3661, 5, 5424, 156, 239, 5192, 2044, 1198, 2, 5887, 1597, 3760, 1615, 1284, 6026, 2912, 464, 3159, 7, 5494, 26, 2452, 1286, 21, 251, 2175, 7031, 14, 183, 4225, 5166, 8, 747, 5633, 3107, 1837, 1084, 1746, 4, 851, 3500, 4307, 264, 7609, 1726, 0, 1, 10, 2033, 1389, 4, 7480, 4676, 1381, 684, 2513, 2518, 8] payload_rows_per_dst=[16256, 14906, 15034, 15800, 15982, 15558, 16308, 14678] dup_rows_per_dst=[21462, 13604, 14548, 19598, 17490, 17314, 23110, 10496] combine_rows_per_src=[16256, 14906, 15034, 15800, 15982, 15558, 16308, 14678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=864600064 dispatch_meta_bytes=882096 combine_payload_bytes=864600064 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=865482160 ingress_bytes=1216921600 link_bytes=1216921600 token_bytes=1730082224 weight_movement_bytes=352321536 total_nvlink_bytes=2082403760 weight_to_token_ratio=0.203644 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=22400 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.814252992e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122096 payload_rows_remote=105898 dup_rows=140048 dup_rows_remote=119380 dedup_ratio=0.534241 router_tokens_per_expert=[4, 11, 1875, 9, 402, 0, 3219, 1, 5753, 245, 174, 5096, 2055, 1105, 0, 6932, 1925, 3807, 1686, 1365, 6304, 3530, 261, 2887, 0, 6051, 19, 1839, 1021, 9, 289, 2338, 7292, 47, 76, 3964, 5620, 9, 525, 4994, 3085, 1249, 816, 1024, 6, 1276, 3933, 4106, 154, 7522, 1423, 7, 6, 12, 3008, 1073, 0, 7530, 5233, 1296, 737, 2805, 2013, 19] payload_rows_per_dst=[16198, 16254, 16228, 14380, 13276, 14400, 16352, 15008] dup_rows_per_dst=[20668, 28786, 28148, 8752, 5954, 13424, 25118, 9198] combine_rows_per_src=[16198, 16254, 16228, 14380, 13276, 14400, 16352, 15008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=872464384 dispatch_meta_bytes=849256 combine_payload_bytes=872464384 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=873313640 ingress_bytes=1224785920 link_bytes=1224785920 token_bytes=1745778024 weight_movement_bytes=352321536 total_nvlink_bytes=2098099560 weight_to_token_ratio=0.201813 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=19712 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.858289901e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125682 payload_rows_remote=109568 dup_rows=136462 dup_rows_remote=114866 dedup_ratio=0.520561 router_tokens_per_expert=[25, 21, 1666, 7, 475, 2, 3290, 1, 5511, 149, 260, 5036, 1760, 836, 1, 6304, 2058, 2414, 2333, 1355, 5987, 3200, 392, 2912, 1, 5406, 49, 2569, 1059, 19, 304, 2521, 7272, 27, 85, 4377, 6035, 14, 856, 5021, 4072, 1617, 1223, 900, 5, 1068, 4233, 5185, 314, 7350, 1870, 0, 8, 10, 1717, 947, 2, 7286, 4804, 1146, 926, 2374, 2388, 17] payload_rows_per_dst=[16010, 14908, 15118, 15792, 16114, 15916, 15658, 16166] dup_rows_per_dst=[18008, 12198, 14210, 19266, 21596, 20690, 8774, 21720] combine_rows_per_src=[16010, 14908, 15118, 15792, 16114, 15916, 15658, 16166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=890650624 dispatch_meta_bytes=906984 combine_payload_bytes=890650624 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=891557608 ingress_bytes=1066811392 link_bytes=1066811392 token_bytes=1782208232 weight_movement_bytes=176160768 total_nvlink_bytes=1958369000 weight_to_token_ratio=0.098844 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=22912 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.973698486e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=867516416 dispatch_meta_bytes=901112 combine_payload_bytes=867516416 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=868417528 ingress_bytes=1219837952 link_bytes=1219837952 token_bytes=1735933944 weight_movement_bytes=352321536 total_nvlink_bytes=2088255480 weight_to_token_ratio=0.202958 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.018939 gemm_active_groups=22 gemm_peak_group_rows=28800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.830583346e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131026 payload_rows_remote=114642 dup_rows=131118 dup_rows_remote=114634 dedup_ratio=0.500175 router_tokens_per_expert=[1, 0, 3, 0, 3, 2, 7, 8191, 0, 8018, 2, 8179, 9, 28, 8184, 0, 35, 2, 0, 8171, 13, 18, 2, 1, 15, 0, 8187, 0, 8188, 3628, 0, 10, 8192, 4571, 9, 0, 1, 111, 8167, 8189, 4, 2, 0, 8189, 8167, 0, 0, 0, 1, 8177, 8188, 7, 0, 0, 1, 1, 0, 6, 0, 0, 1, 8190, 1, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16346, 16382, 16382, 16384] dup_rows_per_dst=[16402, 16084, 16484, 23670, 9374, 16342, 16368, 16394] combine_rows_per_src=[16384, 16380, 16384, 16384, 16346, 16382, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 0, 8175, 0, 8187, 1, 100, 8192, 9, 4, 0, 0, 8183, 4, 0, 0, 0, 3, 0, 8192, 0, 8192, 3501, 0, 0, 8191, 4570, 9, 0, 0, 30, 8185, 8190, 1, 0, 0, 8191, 8191, 0, 0, 1, 0, 8192, 8188, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16564, 16380, 7008, 25588, 16384, 16376, 16392] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114706 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 4, 8166, 3, 8189, 0, 15, 8185, 4, 81, 0, 0, 8143, 15, 2, 0, 0, 7, 1, 8192, 0, 8192, 4838, 0, 0, 8191, 3218, 63, 0, 0, 9, 8181, 8192, 3, 0, 0, 8192, 8192, 0, 0, 11, 0, 8192, 8180, 0, 0, 0, 4, 0, 0, 0, 2, 23, 0, 8190, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16382, 16470, 9692, 22940, 16412, 16368, 16430] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114682 dup_rows=131090 dup_rows_remote=114858 dedup_ratio=0.500069 router_tokens_per_expert=[2, 0, 4, 0, 0, 1, 0, 8188, 0, 8132, 2, 8178, 6, 10, 8151, 2, 3, 0, 2, 8164, 8, 0, 0, 5, 10, 0, 8184, 6, 8188, 2606, 3, 7, 8184, 5576, 70, 0, 0, 55, 8178, 8175, 14, 0, 0, 8185, 8179, 0, 5, 0, 2, 8181, 8188, 16, 0, 0, 3, 0, 1, 7, 0, 4, 0, 8186, 0, 1] payload_rows_per_dst=[16384, 16372, 16384, 16384, 16384, 16380, 16384, 16382] dup_rows_per_dst=[16344, 16232, 16354, 21600, 11390, 16386, 16396, 16388] combine_rows_per_src=[16384, 16372, 16384, 16384, 16384, 16380, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=897581056 dispatch_meta_bytes=897736 combine_payload_bytes=897581056 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=898478792 ingress_bytes=1426063360 link_bytes=1426063360 token_bytes=1796059848 weight_movement_bytes=528482304 total_nvlink_bytes=2324542152 weight_to_token_ratio=0.294245 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=19840 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.985359548e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130948 payload_rows_remote=114564 dup_rows=131196 dup_rows_remote=114758 dedup_ratio=0.500473 router_tokens_per_expert=[2, 0, 0, 0, 7, 3, 6, 8185, 3, 8028, 55, 8174, 3, 126, 8169, 1, 24, 0, 2, 8165, 7, 0, 2, 0, 9, 8, 8170, 27, 8186, 3272, 0, 4, 8155, 4748, 34, 2, 0, 146, 8188, 8179, 8, 0, 2, 8183, 8159, 2, 2, 27, 0, 8190, 8184, 0, 1, 0, 1, 0, 5, 1, 0, 18, 4, 8189, 0, 6] payload_rows_per_dst=[16384, 16362, 16382, 16296, 16380, 16380, 16380, 16384] dup_rows_per_dst=[16360, 15992, 16288, 6514, 26660, 16452, 16492, 16438] combine_rows_per_src=[16384, 16362, 16382, 16296, 16380, 16380, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=917104 combine_payload_bytes=939147264 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940064368 ingress_bytes=1115308032 link_bytes=1115308032 token_bytes=1879211632 weight_movement_bytes=176160768 total_nvlink_bytes=2055372400 weight_to_token_ratio=0.093742 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245259427e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=940441600 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965696 weight_movement_bytes=264241152 total_nvlink_bytes=2144206848 weight_to_token_ratio=0.140556 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917568 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425280 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932992 weight_movement_bytes=176160768 total_nvlink_bytes=2056093760 weight_to_token_ratio=0.093706 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=918160 combine_payload_bytes=939474944 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940393104 ingress_bytes=1467957248 link_bytes=1467957248 token_bytes=1879868048 weight_movement_bytes=528482304 total_nvlink_bytes=2408350352 weight_to_token_ratio=0.281127 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219947833e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938508288 dispatch_meta_bytes=917288 combine_payload_bytes=938508288 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939425576 ingress_bytes=1114669056 link_bytes=1114669056 token_bytes=1877933864 weight_movement_bytes=176160768 total_nvlink_bytes=2054094632 weight_to_token_ratio=0.093806 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.241681428e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130990 payload_rows_remote=114610 dup_rows=131154 dup_rows_remote=114678 dedup_ratio=0.500313 router_tokens_per_expert=[22, 0, 1, 0, 1, 0, 1, 8188, 1, 8152, 23, 8166, 0, 1, 8184, 0, 0, 2, 0, 8179, 14, 4, 4, 0, 25, 0, 8191, 1, 8189, 5771, 16, 2, 8174, 2448, 20, 0, 0, 11, 8183, 8189, 3, 1, 0, 8175, 8186, 3, 2, 0, 0, 8158, 8159, 11, 0, 3, 11, 3, 1, 0, 0, 0, 2, 8189, 2, 0] payload_rows_per_dst=[16380, 16384, 16376, 16384, 16338, 16380, 16366, 16382] dup_rows_per_dst=[16476, 17240, 16336, 27232, 4976, 16360, 16324, 16210] combine_rows_per_src=[16380, 16384, 16376, 16384, 16338, 16380, 16366, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=111582 dedup_ratio=0.500008 router_tokens_per_expert=[2, 1, 0, 0, 0, 1, 9, 8188, 0, 8174, 0, 8176, 0, 28, 8141, 4, 15, 0, 0, 8188, 0, 1, 0, 3, 7, 1, 8192, 0, 8190, 3070, 2, 1, 8186, 5174, 5, 1, 7, 1, 8163, 8191, 2, 0, 0, 8192, 8173, 0, 0, 4, 0, 8192, 8190, 0, 0, 2, 0, 0, 4, 0, 0, 5, 0, 8185, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16378, 16312, 16392, 19492, 13372, 16358, 16384, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114678 dup_rows=131106 dup_rows_remote=114220 dedup_ratio=0.500130 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 3, 8188, 0, 8136, 2, 8187, 6, 16, 8181, 1, 15, 0, 0, 8164, 14, 23, 0, 1, 4, 26, 8187, 2, 8187, 5681, 1, 1, 8150, 2538, 3, 1, 3, 7, 8167, 8178, 9, 0, 0, 8185, 8186, 1, 2, 5, 1, 8185, 8177, 10, 4, 4, 0, 5, 3, 7, 3, 22, 1, 8187, 0, 1] payload_rows_per_dst=[16384, 16380, 16380, 16384, 16360, 16382, 16384, 16384] dup_rows_per_dst=[16356, 16310, 16382, 15952, 16886, 16394, 16388, 16438] combine_rows_per_src=[16384, 16380, 16380, 16384, 16360, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938885120 dispatch_meta_bytes=917152 combine_payload_bytes=938885120 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939802272 ingress_bytes=1291206656 link_bytes=1291206656 token_bytes=1878687392 weight_movement_bytes=352321536 total_nvlink_bytes=2231008928 weight_to_token_ratio=0.187536 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230218297e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=905072 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940412784 ingress_bytes=1467990016 link_bytes=1467990016 token_bytes=1879920496 weight_movement_bytes=528482304 total_nvlink_bytes=2408402800 weight_to_token_ratio=0.281119 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220131320e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=119974 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 8183, 8192, 1, 0, 4, 8138, 81, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 2, 7, 8124, 0, 0, 0, 0, 8192, 8192, 0, 8148, 0, 5593, 0, 0, 8186, 0, 0, 8192, 0, 0, 8192, 0, 0, 82, 8192, 0, 2679, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16268, 16546, 16384, 16384, 16114, 11100, 16372, 21906] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114706 dedup_ratio=0.500061 router_tokens_per_expert=[1, 0, 8086, 8189, 2, 5, 18, 8061, 435, 8190, 0, 0, 0, 1, 7, 0, 0, 0, 0, 1, 4, 0, 8187, 2, 8189, 0, 0, 1, 0, 0, 0, 2, 13, 0, 8044, 6, 0, 8167, 0, 1, 0, 2, 8181, 8186, 0, 8182, 1, 4546, 0, 17, 8177, 0, 0, 8191, 1, 9, 8189, 3, 1, 72, 8191, 1, 3509, 1] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16372, 16382, 16384, 16384] dup_rows_per_dst=[15962, 17244, 16382, 16374, 16090, 25458, 16406, 7172] combine_rows_per_src=[16384, 16384, 16384, 16382, 16372, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114714 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 8177, 8192, 0, 0, 4, 8138, 13, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8164, 16, 37, 8136, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 5409, 0, 0, 8177, 0, 0, 8192, 0, 3, 8192, 0, 0, 24, 8192, 0, 2852, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16254, 16412, 16384, 16384, 16326, 10818, 16360, 22136] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114202 dedup_ratio=0.500000 router_tokens_per_expert=[0, 2, 7963, 8186, 0, 0, 35, 8056, 242, 8191, 1, 1, 1, 0, 0, 0, 0, 5, 1, 0, 1, 0, 8192, 0, 8192, 3, 1, 0, 0, 0, 0, 0, 0, 0, 7886, 15, 14, 7970, 1, 0, 0, 2, 8191, 8182, 0, 8191, 0, 6295, 1, 10, 8182, 0, 0, 8191, 0, 0, 8191, 0, 0, 6, 8182, 0, 2489, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15728, 16870, 16386, 16372, 15388, 28974, 16384, 4970] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 4, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=103258 dedup_ratio=0.500092 router_tokens_per_expert=[1, 5, 8129, 8182, 0, 1, 41, 7450, 260, 8186, 0, 6, 4, 5, 0, 7, 0, 5, 0, 4, 1, 0, 8188, 0, 8183, 0, 4, 0, 4, 1, 0, 0, 40, 0, 8125, 36, 0, 8182, 4, 2, 0, 1, 8178, 8187, 1, 8164, 5, 2972, 0, 9, 8185, 2, 0, 8190, 4, 7, 8188, 3, 0, 17, 8180, 2, 5720, 1] payload_rows_per_dst=[16372, 16384, 16384, 16384, 16384, 16372, 16384, 16384] dup_rows_per_dst=[14858, 13982, 16400, 16374, 16394, 8840, 16410, 27838] combine_rows_per_src=[16372, 16384, 16384, 16384, 16384, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=915592 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940357768 ingress_bytes=1556004864 link_bytes=1556004864 token_bytes=1879799944 weight_movement_bytes=616562688 total_nvlink_bytes=2496362632 weight_to_token_ratio=0.327994 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.712977730e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=938648 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940462744 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879986840 weight_movement_bytes=704643072 total_nvlink_bytes=2584629912 weight_to_token_ratio=0.374813 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917512 combine_payload_bytes=939393024 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940310536 ingress_bytes=1203634176 link_bytes=1203634176 token_bytes=1879703560 weight_movement_bytes=264241152 total_nvlink_bytes=2143944712 weight_to_token_ratio=0.140576 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739848964e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=915560 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940439656 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879963752 weight_movement_bytes=176160768 total_nvlink_bytes=2056124520 weight_to_token_ratio=0.093704 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917600 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425312 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879933024 weight_movement_bytes=176160768 total_nvlink_bytes=2056093792 weight_to_token_ratio=0.093706 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=871688 combine_payload_bytes=939327488 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=940199176 ingress_bytes=1643970560 link_bytes=1643970560 token_bytes=1879526664 weight_movement_bytes=704643072 total_nvlink_bytes=2584169736 weight_to_token_ratio=0.374905 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205548909e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114680 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8058, 8191, 0, 0, 15, 8135, 93, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 2, 8071, 38, 27, 8189, 0, 0, 0, 7, 8190, 8192, 0, 8192, 0, 3310, 0, 0, 8181, 0, 0, 8192, 0, 1, 8191, 0, 0, 56, 8192, 0, 4969, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16032, 16564, 16384, 16392, 16270, 12362, 16364, 20704] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127668 payload_rows_remote=111284 dup_rows=134476 dup_rows_remote=114712 dedup_ratio=0.512985 router_tokens_per_expert=[1, 0, 8175, 8183, 1, 1, 31, 7892, 141, 8190, 3, 4, 0, 4, 0, 1, 0, 0, 0, 3, 1, 1, 8188, 0, 8190, 6, 0, 0, 0, 0, 1, 0, 91, 1, 8112, 30, 1, 8089, 1, 3, 1, 6, 8177, 8186, 0, 8185, 2, 3842, 0, 3, 8178, 0, 0, 8189, 1, 0, 8189, 1, 7, 5, 8188, 3, 4561, 3] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16380, 12986] dup_rows_per_dst=[19764, 16656, 16354, 16410, 18246, 24038, 16422, 6586] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16380, 12986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 3, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114822 dedup_ratio=0.500023 router_tokens_per_expert=[1, 1, 8130, 8190, 6, 1, 17, 8145, 211, 8190, 0, 2, 0, 0, 1, 0, 1, 0, 14, 0, 0, 1, 8186, 0, 8188, 2, 0, 2, 0, 0, 1, 0, 3, 0, 8092, 16, 26, 8182, 0, 1, 5, 1, 8187, 8191, 1, 8187, 1, 4747, 2, 3, 8178, 1, 1, 8182, 0, 4, 8190, 1, 4, 5, 8189, 0, 3382, 0] payload_rows_per_dst=[16384, 16382, 16384, 16382, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16218, 16800, 16400, 16386, 16256, 9500, 16358, 23160] combine_rows_per_src=[16384, 16382, 16384, 16382, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917472 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441568 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965664 weight_movement_bytes=352321536 total_nvlink_bytes=2232287200 weight_to_token_ratio=0.187408 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=911638528 dispatch_meta_bytes=903984 combine_payload_bytes=911638528 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=912542512 ingress_bytes=1352040448 link_bytes=1352040448 token_bytes=1824181040 weight_movement_bytes=440401920 total_nvlink_bytes=2264582960 weight_to_token_ratio=0.241424 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.570862139e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=115774 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 8156, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 7502, 3, 0, 0, 8134, 0, 2844, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 0, 0, 2, 0, 8162, 0, 0, 7677, 8192, 5474, 8192, 0, 0, 2, 8192, 8192, 0, 1213, 0, 8192, 0, 0, 3, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16312, 16364, 14894, 5674, 16378, 15298, 27336, 18816] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=116822 dedup_ratio=0.500015 router_tokens_per_expert=[0, 3, 1, 0, 8147, 0, 1, 0, 8190, 2, 0, 3, 1, 0, 8158, 0, 0, 1, 7142, 3, 1, 1, 8168, 3, 3459, 3, 2, 8188, 1, 4, 0, 0, 8, 0, 8187, 0, 8191, 8, 0, 1, 0, 0, 0, 8148, 5, 0, 7203, 8190, 2992, 8184, 2, 4, 1, 8190, 8189, 0, 3884, 6, 8190, 2, 0, 2, 0, 3] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[16298, 16324, 14254, 23310, 16406, 22600, 5982, 15902] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114704 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 8158, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 5216, 28, 0, 0, 8069, 0, 5628, 0, 0, 8098, 0, 0, 5, 0, 22, 0, 8167, 0, 8192, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 7673, 8192, 4203, 8191, 0, 0, 44, 8192, 8192, 0, 2039, 0, 8192, 0, 0, 2, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16316, 16372, 15606, 27462, 16464, 26282, 8492, 4082] combine_rows_per_src=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 2, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130964 payload_rows_remote=114580 dup_rows=131180 dup_rows_remote=107860 dedup_ratio=0.500412 router_tokens_per_expert=[0, 4, 0, 8, 8070, 0, 5, 15, 8178, 23, 1, 3, 5, 0, 8162, 1, 0, 0, 7342, 14, 6, 14, 8031, 9, 3656, 0, 0, 8156, 2, 4, 2, 0, 8, 5, 8160, 6, 8186, 5, 2, 0, 0, 0, 4, 8135, 0, 0, 7774, 8186, 2655, 8173, 4, 5, 11, 8186, 8187, 3, 3460, 0, 8186, 8, 2, 10, 0, 0] payload_rows_per_dst=[16384, 16368, 16368, 16376, 16384, 16316, 16384, 16384] dup_rows_per_dst=[16194, 16378, 14464, 7264, 16360, 8760, 28440, 23320] combine_rows_per_src=[16384, 16368, 16368, 16376, 16384, 16316, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130974 payload_rows_remote=114590 dup_rows=131170 dup_rows_remote=104996 dedup_ratio=0.500374 router_tokens_per_expert=[0, 0, 0, 0, 8129, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 8147, 0, 0, 0, 6191, 27, 0, 0, 8085, 0, 4288, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 8163, 0, 8192, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 7491, 8192, 4873, 8192, 0, 0, 22, 8192, 8192, 0, 1936, 1, 8192, 0, 0, 38, 0, 3] payload_rows_per_dst=[16384, 16384, 16286, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16258, 16300, 12320, 8482, 16326, 14970, 26174, 20340] combine_rows_per_src=[16384, 16384, 16286, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=918016 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940392960 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879867904 weight_movement_bytes=176160768 total_nvlink_bytes=2056028672 weight_to_token_ratio=0.093709 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=926024 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940417352 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879908680 weight_movement_bytes=176160768 total_nvlink_bytes=2056069448 weight_to_token_ratio=0.093707 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=921848 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=940445944 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879970040 weight_movement_bytes=880803840 total_nvlink_bytes=2760773880 weight_to_token_ratio=0.468520 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=28672 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[4, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917560 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425272 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932984 weight_movement_bytes=176160768 total_nvlink_bytes=2056093752 weight_to_token_ratio=0.093706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938639360 dispatch_meta_bytes=889760 combine_payload_bytes=938639360 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939529120 ingress_bytes=1114800128 link_bytes=1114800128 token_bytes=1878168480 weight_movement_bytes=176160768 total_nvlink_bytes=2054329248 weight_to_token_ratio=0.093794 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.242415376e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938721280 dispatch_meta_bytes=878344 combine_payload_bytes=938721280 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939599624 ingress_bytes=1643364352 link_bytes=1643364352 token_bytes=1878320904 weight_movement_bytes=704643072 total_nvlink_bytes=2582963976 weight_to_token_ratio=0.375145 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202154397e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114656 dup_rows=131104 dup_rows_remote=113364 dedup_ratio=0.500122 router_tokens_per_expert=[0, 0, 0, 0, 8149, 0, 0, 4, 8192, 0, 0, 0, 0, 0, 8144, 0, 0, 1, 6314, 27, 0, 0, 8114, 0, 4779, 0, 0, 8157, 0, 0, 0, 0, 18, 0, 8173, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6973, 8192, 2708, 8192, 0, 0, 26, 8192, 8192, 0, 3947, 0, 8192, 0, 0, 1, 0, 1] payload_rows_per_dst=[16384, 16384, 16352, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16306, 16288, 12560, 17740, 16382, 13946, 13600, 24282] combine_rows_per_src=[16384, 16384, 16352, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130594 payload_rows_remote=114210 dup_rows=131550 dup_rows_remote=114514 dedup_ratio=0.501823 router_tokens_per_expert=[0, 3, 0, 2, 8179, 0, 3, 0, 8188, 0, 0, 6, 3, 0, 8109, 6, 0, 0, 7893, 30, 2, 0, 8140, 0, 6609, 4, 1, 8167, 0, 5, 3, 4, 6, 3, 8180, 0, 8189, 1, 1, 0, 0, 10, 5, 8167, 0, 2, 7507, 8186, 1663, 8185, 3, 0, 0, 8188, 8188, 0, 968, 11, 8183, 4, 0, 57, 5, 3] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16384, 16384, 15912, 16384] dup_rows_per_dst=[17036, 16838, 31892, 29578, 16422, 14586, 3120, 2078] combine_rows_per_src=[16384, 16384, 16378, 16384, 16384, 16384, 15912, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=912080 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940174032 ingress_bytes=1115422720 link_bytes=1115422720 token_bytes=1879435984 weight_movement_bytes=176160768 total_nvlink_bytes=2055596752 weight_to_token_ratio=0.093731 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245901632e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=935608320 dispatch_meta_bytes=914896 combine_payload_bytes=935608320 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936523216 ingress_bytes=1287929856 link_bytes=1287929856 token_bytes=1872131536 weight_movement_bytes=352321536 total_nvlink_bytes=2224453072 weight_to_token_ratio=0.188193 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.211869585e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114682 dup_rows=131088 dup_rows_remote=114366 dedup_ratio=0.500061 router_tokens_per_expert=[6, 0, 7329, 8188, 0, 0, 2, 2, 8190, 0, 8186, 0, 0, 2, 1, 2, 0, 8182, 1, 5, 8172, 0, 187, 1, 2, 8191, 0, 4, 8188, 0, 4, 8188, 8188, 8190, 3, 0, 8189, 3, 0, 8190, 547, 3, 0, 2, 8188, 2, 0, 3, 2, 10, 1, 0, 8189, 0, 0, 1, 0, 2, 133, 4, 8178, 1, 6, 4] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16380, 16382, 16384, 16384] dup_rows_per_dst=[14670, 16378, 16722, 16394, 16388, 17484, 16402, 16650] combine_rows_per_src=[16384, 16384, 16374, 16384, 16380, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114202 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8030, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8096, 0, 1, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 8192, 243, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 8192, 0, 0, 0, 5, 0, 4, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16060, 16384, 16194, 16384, 16384, 16870, 16394, 16402] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114658 dup_rows=131102 dup_rows_remote=114754 dedup_ratio=0.500114 router_tokens_per_expert=[0, 0, 7383, 8192, 0, 0, 0, 0, 8174, 0, 8192, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 8046, 0, 25, 0, 0, 8189, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 14, 0, 8192, 0, 1, 8169, 807, 0, 0, 0, 8192, 0, 0, 0, 0, 132, 0, 0, 8192, 0, 1, 0, 9, 0, 17, 0, 8191, 0, 10, 0] payload_rows_per_dst=[16384, 16384, 16354, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14766, 16348, 16140, 16378, 16368, 17998, 16650, 16454] combine_rows_per_src=[16384, 16384, 16354, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114640 dup_rows=131120 dup_rows_remote=114786 dedup_ratio=0.500183 router_tokens_per_expert=[0, 0, 1, 1, 8174, 0, 1, 0, 8192, 5, 0, 3, 0, 0, 8104, 1, 1, 4, 7717, 8, 1, 0, 8078, 1, 5815, 0, 1, 8160, 0, 1, 0, 0, 8, 0, 8164, 2, 8183, 1, 1, 0, 0, 0, 0, 8183, 0, 1, 7359, 8191, 968, 8191, 0, 0, 43, 8189, 8191, 1, 2917, 4, 8188, 0, 0, 3, 0, 15] payload_rows_per_dst=[16384, 16384, 16348, 16384, 16384, 16374, 16382, 16384] dup_rows_per_dst=[16352, 16226, 15272, 11570, 16334, 14712, 18402, 22252] combine_rows_per_src=[16384, 16384, 16348, 16384, 16384, 16374, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114478 dedup_ratio=0.500069 router_tokens_per_expert=[8, 1, 7709, 8186, 0, 4, 6, 5, 8184, 0, 8182, 4, 0, 5, 3, 1, 0, 8179, 2, 8, 7968, 0, 52, 0, 10, 8183, 0, 4, 8186, 6, 0, 8181, 8183, 8185, 1, 0, 8185, 15, 3, 8187, 520, 5, 0, 6, 8186, 4, 6, 1, 0, 7, 4, 0, 8174, 0, 2, 6, 90, 5, 11, 0, 8187, 4, 13, 5] payload_rows_per_dst=[16384, 16384, 16380, 16380, 16376, 16384, 16382, 16384] dup_rows_per_dst=[15454, 16374, 16038, 16354, 16440, 17444, 16374, 16612] combine_rows_per_src=[16384, 16384, 16380, 16380, 16376, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114682 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8114, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8116, 0, 1, 0, 4, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 1, 0, 8186, 94, 1, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 8192, 0, 0, 0, 53, 0, 3, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16228, 16384, 16234, 16392, 16374, 16574, 16390, 16496] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=916192 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940391136 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879866080 weight_movement_bytes=176160768 total_nvlink_bytes=2056026848 weight_to_token_ratio=0.093709 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=915560 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940439656 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879963752 weight_movement_bytes=176160768 total_nvlink_bytes=2056124520 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=917648 combine_payload_bytes=939278336 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940195984 ingress_bytes=1115439104 link_bytes=1115439104 token_bytes=1879474320 weight_movement_bytes=176160768 total_nvlink_bytes=2055635088 weight_to_token_ratio=0.093729 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245993375e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=917704 combine_payload_bytes=939130880 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940048584 ingress_bytes=1115291648 link_bytes=1115291648 token_bytes=1879179464 weight_movement_bytes=176160768 total_nvlink_bytes=2055340232 weight_to_token_ratio=0.093743 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245167683e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=916592 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940293232 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879669872 weight_movement_bytes=176160768 total_nvlink_bytes=2055830640 weight_to_token_ratio=0.093719 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917480 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441576 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965672 weight_movement_bytes=176160768 total_nvlink_bytes=2056126440 weight_to_token_ratio=0.093704 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114702 dedup_ratio=0.500053 router_tokens_per_expert=[0, 0, 7751, 8192, 0, 0, 0, 0, 8183, 0, 8192, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 8140, 0, 8, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 3, 0, 8191, 1, 0, 8185, 461, 0, 0, 0, 8192, 0, 0, 0, 0, 28, 0, 0, 8192, 0, 0, 0, 5, 0, 4, 0, 8192, 0, 15, 0] payload_rows_per_dst=[16384, 16384, 16370, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15502, 16366, 16280, 16384, 16376, 17306, 16440, 16432] combine_rows_per_src=[16384, 16384, 16370, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=107514 dedup_ratio=0.500069 router_tokens_per_expert=[0, 0, 7990, 8188, 2, 4, 4, 4, 8189, 7, 8185, 1, 0, 4, 0, 0, 0, 8156, 3, 6, 7909, 0, 24, 1, 6, 8188, 0, 0, 8188, 3, 0, 8185, 8187, 8187, 0, 0, 8188, 0, 0, 8188, 398, 4, 0, 3, 8186, 3, 0, 2, 2, 44, 4, 0, 8185, 0, 0, 4, 5, 2, 9, 14, 8187, 3, 15, 5] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16376, 16384, 16378, 16384] dup_rows_per_dst=[23576, 16468, 17224, 16356, 16482, 9556, 15936, 15492] combine_rows_per_src=[16384, 16384, 16384, 16380, 16376, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917504 combine_payload_bytes=939409408 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=940326912 ingress_bytes=1644052480 link_bytes=1644052480 token_bytes=1879736320 weight_movement_bytes=704643072 total_nvlink_bytes=2584379392 weight_to_token_ratio=0.374863 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206007627e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130224 payload_rows_remote=113842 dup_rows=131920 dup_rows_remote=115564 dedup_ratio=0.503235 router_tokens_per_expert=[4, 4, 1, 0, 3, 2, 0, 3350, 8189, 0, 8186, 2, 1351, 0, 0, 1, 0, 4, 1, 8174, 0, 2, 8186, 2, 8188, 4, 0, 0, 8176, 1, 8182, 3, 8188, 8188, 4225, 2, 0, 0, 6, 0, 7463, 1, 3, 8189, 0, 8189, 0, 8188, 2, 1, 2, 8185, 1, 3, 8, 3, 0, 3, 3, 0, 8188, 2, 1, 12] payload_rows_per_dst=[16384, 16382, 16382, 16374, 16384, 15550, 16384, 16384] dup_rows_per_dst=[23098, 19076, 16356, 16358, 16646, 7574, 16402, 16410] combine_rows_per_src=[16384, 16382, 16382, 16374, 16384, 15550, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=888736 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940265376 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879642016 weight_movement_bytes=176160768 total_nvlink_bytes=2055802784 weight_to_token_ratio=0.093720 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=105532 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 2784, 8192, 0, 8192, 0, 964, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 5162, 0, 0, 0, 0, 0, 7469, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[11352, 10680, 16382, 16384, 17958, 25540, 16384, 16392] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128178 payload_rows_remote=111794 dup_rows=133966 dup_rows_remote=110838 dedup_ratio=0.511040 router_tokens_per_expert=[0, 19, 5, 0, 0, 0, 10, 4412, 8189, 0, 8192, 0, 2194, 0, 0, 17, 0, 0, 0, 8187, 0, 0, 8192, 0, 8192, 0, 1, 0, 8189, 0, 8192, 0, 8192, 8185, 3073, 0, 0, 0, 0, 0, 6690, 0, 0, 8192, 0, 8174, 0, 8192, 0, 0, 0, 8171, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 0, 11] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16374, 13500, 16384, 16384] dup_rows_per_dst=[25276, 23128, 16374, 16380, 14324, 5718, 16342, 16424] combine_rows_per_src=[16384, 16384, 16384, 16384, 16374, 13500, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114796 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 7407, 8191, 0, 0, 1, 1, 8186, 0, 8191, 0, 0, 0, 0, 0, 2, 8186, 1, 0, 8150, 1, 1, 0, 32, 8186, 0, 1, 8190, 0, 1, 8191, 8191, 8191, 6, 3, 8189, 2, 1, 8131, 730, 1, 0, 1, 8191, 0, 0, 0, 0, 54, 0, 1, 8191, 1, 0, 1, 14, 0, 55, 0, 8192, 0, 7, 0] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14816, 16372, 16300, 16438, 16280, 17842, 16494, 16534] combine_rows_per_src=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[4, 1, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130956 payload_rows_remote=114572 dup_rows=131188 dup_rows_remote=114802 dedup_ratio=0.500443 router_tokens_per_expert=[6, 17, 18, 0, 6, 5, 4, 4697, 8185, 1, 8179, 6, 325, 0, 0, 11, 0, 7, 0, 8184, 0, 9, 8186, 6, 8186, 13, 0, 0, 8179, 2, 8183, 8, 8186, 8171, 4301, 4, 0, 0, 10, 0, 7067, 2, 0, 8186, 0, 8148, 0, 8183, 3, 4, 0, 8174, 0, 4, 1, 6, 12, 5, 6, 0, 8173, 0, 2, 1] payload_rows_per_dst=[16384, 16274, 16384, 16384, 16382, 16380, 16384, 16384] dup_rows_per_dst=[9494, 674, 16414, 16400, 16772, 38608, 16440, 16386] combine_rows_per_src=[16384, 16274, 16384, 16384, 16382, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114690 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 26, 3003, 8192, 0, 8192, 0, 3200, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 8192, 0, 8192, 0, 0, 0, 8190, 0, 8191, 0, 8192, 8180, 5070, 0, 0, 0, 0, 0, 5072, 0, 0, 8192, 0, 8172, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 37, 0, 0, 0, 8192, 0, 0, 11] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[6058, 10790, 16380, 16380, 18308, 30294, 16384, 16480] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=932593664 dispatch_meta_bytes=917624 combine_payload_bytes=932593664 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=933511288 ingress_bytes=1108754432 link_bytes=1108754432 token_bytes=1866104952 weight_movement_bytes=176160768 total_nvlink_bytes=2042265720 weight_to_token_ratio=0.094400 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.208562002e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=880880 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=704643072 egress_bytes=940404976 ingress_bytes=1732247552 link_bytes=1732247552 token_bytes=1879929072 weight_movement_bytes=792723456 total_nvlink_bytes=2672652528 weight_to_token_ratio=0.421677 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=28800 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699863215e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=915816448 dispatch_meta_bytes=890528 combine_payload_bytes=915816448 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=916706976 ingress_bytes=1356218368 link_bytes=1356218368 token_bytes=1832523424 weight_movement_bytes=440401920 total_nvlink_bytes=2272925344 weight_to_token_ratio=0.240325 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.594256747e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917920 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940409248 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=1879900576 weight_movement_bytes=704643072 total_nvlink_bytes=2584543648 weight_to_token_ratio=0.374830 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938573824 dispatch_meta_bytes=917496 combine_payload_bytes=938573824 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939491320 ingress_bytes=1114734592 link_bytes=1114734592 token_bytes=1878065144 weight_movement_bytes=176160768 total_nvlink_bytes=2054225912 weight_to_token_ratio=0.093799 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.242048402e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940425216 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932928 weight_movement_bytes=352321536 total_nvlink_bytes=2232254464 weight_to_token_ratio=0.187412 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130086 payload_rows_remote=113702 dup_rows=132058 dup_rows_remote=115692 dedup_ratio=0.503761 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 8, 2126, 8192, 3, 8192, 0, 1643, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 8183, 0, 8192, 0, 8192, 8189, 5303, 0, 0, 0, 0, 0, 7331, 0, 0, 8192, 0, 8160, 0, 8192, 0, 0, 0, 8178, 0, 0, 0, 0, 6, 0, 0, 0, 8192, 0, 0, 17] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 15402, 16384, 16384] dup_rows_per_dst=[20660, 19678, 16384, 16366, 13750, 12434, 16356, 16430] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 15402, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129390 payload_rows_remote=113006 dup_rows=132754 dup_rows_remote=114822 dedup_ratio=0.506416 router_tokens_per_expert=[5, 3, 1, 0, 0, 2, 3, 770, 8188, 0, 8185, 6, 1960, 0, 0, 0, 0, 6, 1, 8180, 0, 4, 8184, 4, 8185, 8, 0, 0, 8187, 5, 8182, 9, 8187, 8185, 6268, 4, 0, 0, 10, 0, 7384, 0, 0, 8187, 1, 8183, 0, 8187, 2, 4, 0, 8186, 0, 3, 3, 4, 2, 5, 1, 0, 8184, 1, 0, 3] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 14710, 16384, 16380] dup_rows_per_dst=[17932, 28474, 16396, 16438, 21106, 2, 16390, 16016] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 14710, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114722 dedup_ratio=0.500053 router_tokens_per_expert=[4, 3, 0, 1, 2, 3, 8188, 0, 4, 0, 76, 6, 0, 4, 4879, 4, 0, 8188, 4, 8182, 0, 0, 0, 0, 0, 8185, 0, 8189, 4, 0, 5, 8187, 0, 0, 1, 3, 2, 0, 8189, 0, 3176, 8135, 603, 3692, 4, 3, 4, 0, 0, 8188, 0, 3, 3, 0, 1, 8188, 8188, 3, 2, 8188, 8189, 4016, 8173, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16372, 16384, 16382] dup_rows_per_dst=[16396, 26316, 16364, 16378, 16382, 14862, 16382, 8006] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16372, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=931446784 dispatch_meta_bytes=917576 combine_payload_bytes=931446784 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=932364360 ingress_bytes=1459929088 link_bytes=1459929088 token_bytes=1863811144 weight_movement_bytes=528482304 total_nvlink_bytes=2392293448 weight_to_token_ratio=0.283549 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.174993488e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114688 dup_rows=131090 dup_rows_remote=111354 dedup_ratio=0.500069 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 29, 0, 0, 0, 4015, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 5623, 8143, 150, 4135, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 8192, 2479, 8192, 3] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16366, 16384, 16384] dup_rows_per_dst=[16384, 8088, 16382, 16384, 16384, 19736, 16384, 21348] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16366, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=925745152 dispatch_meta_bytes=911312 combine_payload_bytes=925745152 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=926656464 ingress_bytes=1278066688 link_bytes=1278066688 token_bytes=1852401616 weight_movement_bytes=352321536 total_nvlink_bytes=2204723152 weight_to_token_ratio=0.190197 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.156639962e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131006 payload_rows_remote=114622 dup_rows=131138 dup_rows_remote=106038 dedup_ratio=0.500252 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 79, 0, 0, 0, 4280, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8191, 0, 0, 8, 0, 0, 0, 8192, 0, 2803, 8110, 435, 3589, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 8191, 5263, 8192, 12] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16318, 16384, 16384] dup_rows_per_dst=[16384, 25100, 16384, 16382, 16400, 13556, 16382, 10550] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16318, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131012 payload_rows_remote=114628 dup_rows=131132 dup_rows_remote=105924 dedup_ratio=0.500229 router_tokens_per_expert=[8, 0, 1, 5, 3, 6, 8180, 7, 1, 0, 61, 13, 0, 9, 2162, 7, 0, 8179, 6, 8172, 0, 10, 0, 0, 1, 8181, 0, 8184, 6, 0, 5, 8182, 0, 2, 1, 8, 7, 0, 8186, 0, 3603, 8118, 247, 5978, 1, 6, 1, 5, 0, 8176, 7, 4, 4, 0, 3, 8185, 8185, 6, 7, 8181, 8175, 4395, 8181, 11] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16330, 16382, 16384] dup_rows_per_dst=[16334, 4492, 16350, 16424, 16360, 19588, 16376, 25208] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16330, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114670 dup_rows=131104 dup_rows_remote=118866 dedup_ratio=0.500122 router_tokens_per_expert=[0, 9, 5, 0, 1, 0, 4, 2374, 8191, 0, 8192, 0, 1147, 0, 0, 4, 2, 1, 1, 8192, 0, 0, 8191, 1, 8190, 0, 5, 3, 8182, 1, 8190, 0, 8191, 8183, 6119, 1, 0, 0, 1, 0, 6666, 0, 13, 8191, 1, 8173, 0, 8188, 1, 0, 0, 8187, 0, 0, 0, 1, 63, 1, 0, 1, 8191, 0, 1, 13] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16370, 16370, 16382, 16384] dup_rows_per_dst=[17202, 18684, 16392, 16380, 12238, 17294, 16376, 16538] combine_rows_per_src=[16384, 16384, 16384, 16382, 16370, 16370, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917584 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940326992 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879736400 weight_movement_bytes=176160768 total_nvlink_bytes=2055897168 weight_to_token_ratio=0.093716 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=904168 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940428264 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879952360 weight_movement_bytes=176160768 total_nvlink_bytes=2056113128 weight_to_token_ratio=0.093705 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114700 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 56, 0, 0, 0, 5280, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 1896, 8140, 1519, 3208, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 8173, 4495, 8192, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16372, 16384, 16384] dup_rows_per_dst=[16384, 10672, 16384, 16384, 16384, 13154, 16384, 25338] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=882640 combine_payload_bytes=938983424 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939866064 ingress_bytes=1291304960 link_bytes=1291304960 token_bytes=1878849488 weight_movement_bytes=352321536 total_nvlink_bytes=2231171024 weight_to_token_ratio=0.187520 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230768759e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=12 export_slots=12 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 4, 1, 1, 1, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=882208 combine_payload_bytes=939032576 prefetch_bytes=88080384 reduce_bytes=1056964608 egress_bytes=939914784 ingress_bytes=2084077568 link_bytes=2084077568 token_bytes=1878947360 weight_movement_bytes=1145044992 total_nvlink_bytes=3023992352 weight_to_token_ratio=0.609408 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.166996444e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=934144 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940310784 ingress_bytes=1291698176 link_bytes=1291698176 token_bytes=1879687424 weight_movement_bytes=352321536 total_nvlink_bytes=2232008960 weight_to_token_ratio=0.187436 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232970604e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=114712 dedup_ratio=0.500092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 68, 0, 0, 0, 4570, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 8192, 0, 2136, 8132, 2350, 2805, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8192, 8189, 4514, 8192, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16364, 16384, 16380] dup_rows_per_dst=[16384, 23844, 16382, 16384, 16392, 14482, 16384, 10844] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16364, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130196 payload_rows_remote=113812 dup_rows=131948 dup_rows_remote=115522 dedup_ratio=0.503342 router_tokens_per_expert=[4, 3, 6, 5, 0, 7, 8186, 8, 0, 1, 88, 2, 7, 0, 3626, 5, 0, 8168, 5, 8180, 4, 0, 0, 1, 2, 8185, 0, 8186, 6, 0, 8, 8187, 0, 0, 0, 5, 5, 0, 8187, 0, 3968, 8159, 78, 3968, 5, 4, 0, 0, 0, 8186, 0, 3, 4, 0, 0, 8187, 8187, 5, 4, 8184, 8183, 4685, 8165, 20] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16356, 16380, 15540] dup_rows_per_dst=[16426, 23814, 16448, 16356, 16360, 18244, 16402, 7898] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16356, 16380, 15540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917504 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940343296 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879769088 weight_movement_bytes=176160768 total_nvlink_bytes=2055929856 weight_to_token_ratio=0.093714 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130836 payload_rows_remote=114452 dup_rows=131308 dup_rows_remote=106738 dedup_ratio=0.500900 router_tokens_per_expert=[0, 0, 0, 18, 0, 2, 3326, 0, 4, 8188, 4, 4, 0, 56, 4, 0, 8189, 8187, 8188, 4, 0, 8187, 1, 0, 0, 8165, 0, 8188, 3, 0, 0, 3, 2, 8188, 8186, 0, 8188, 2, 2, 8188, 0, 0, 0, 4, 3, 3, 2, 5, 0, 2, 4, 0, 4931, 0, 0, 6, 0, 0, 8186, 4, 0, 8071, 8187, 8187] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16150] dup_rows_per_dst=[23060, 16510, 24570, 16334, 16376, 16402, 14376, 3680] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1722, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 8190, 8192, 8192, 0, 0, 8192, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6479, 0, 0, 0, 0, 0, 8192, 0, 0, 8177, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[5300, 16402, 8342, 16382, 16384, 16384, 21150, 30728] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=932347904 dispatch_meta_bytes=917336 combine_payload_bytes=932347904 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=933265240 ingress_bytes=1108508672 link_bytes=1108508672 token_bytes=1865613144 weight_movement_bytes=176160768 total_nvlink_bytes=2041773912 weight_to_token_ratio=0.094425 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.207185849e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=917504 combine_payload_bytes=939327488 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940244992 ingress_bytes=1379729408 link_bytes=1379729408 token_bytes=1879572480 weight_movement_bytes=440401920 total_nvlink_bytes=2319974400 weight_to_token_ratio=0.234310 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725908757e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 1, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130040 payload_rows_remote=113656 dup_rows=132104 dup_rows_remote=115518 dedup_ratio=0.503937 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2151, 0, 0, 8192, 0, 0, 0, 101, 0, 0, 8163, 8192, 8192, 0, 0, 8191, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8185, 8192, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6495, 0, 0, 0, 0, 0, 8192, 0, 0, 7676, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 15352] dup_rows_per_dst=[20686, 16586, 24516, 16384, 16370, 16380, 21182, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 15352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937590784 dispatch_meta_bytes=884760 combine_payload_bytes=937590784 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=938475544 ingress_bytes=1818394624 link_bytes=1818394624 token_bytes=1876066328 weight_movement_bytes=880803840 total_nvlink_bytes=2756870168 weight_to_token_ratio=0.469495 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018225086e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114654 dup_rows=131106 dup_rows_remote=97954 dedup_ratio=0.500130 router_tokens_per_expert=[6, 0, 0, 18, 0, 6, 1981, 0, 8, 8185, 8, 7, 0, 25, 7, 0, 8184, 8181, 8186, 7, 0, 8177, 0, 3, 0, 8178, 0, 8184, 4, 0, 7, 7, 6, 8185, 8183, 0, 8180, 0, 6, 8180, 0, 0, 2, 4, 6, 4, 0, 9, 0, 5, 8, 0, 6351, 0, 0, 5, 3, 0, 8182, 7, 5, 8009, 8183, 8180] payload_rows_per_dst=[16382, 16384, 16352, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[4006, 15686, 8084, 16512, 16366, 16404, 20896, 33152] combine_rows_per_src=[16382, 16384, 16352, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131032 payload_rows_remote=114648 dup_rows=131112 dup_rows_remote=114702 dedup_ratio=0.500153 router_tokens_per_expert=[1, 0, 0, 1, 0, 1, 8191, 0, 1, 2, 115, 1, 0, 1, 3320, 1, 0, 8190, 1, 8189, 1, 0, 0, 0, 0, 8190, 0, 8190, 0, 3, 0, 8191, 0, 1, 14, 1, 1, 0, 8191, 0, 3472, 8094, 298, 5157, 0, 1, 0, 0, 0, 8191, 0, 0, 2, 0, 0, 8191, 8190, 0, 1, 8187, 8180, 4096, 8191, 23] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16348, 16382, 16384] dup_rows_per_dst=[16380, 8746, 16378, 16386, 16410, 17696, 16386, 22730] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16348, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=120226 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6937, 0, 0, 8192, 0, 0, 0, 53, 0, 0, 8191, 8192, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 0, 0, 0, 8192, 0, 0, 8067, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[13874, 16490, 8190, 16384, 16384, 16386, 10846, 32518] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=931069952 dispatch_meta_bytes=916696 combine_payload_bytes=931069952 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=931986648 ingress_bytes=1107230720 link_bytes=1107230720 token_bytes=1863056600 weight_movement_bytes=176160768 total_nvlink_bytes=2039217368 weight_to_token_ratio=0.094555 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.200029851e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=850432 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940096000 ingress_bytes=1643888640 link_bytes=1643888640 token_bytes=1879341568 weight_movement_bytes=704643072 total_nvlink_bytes=2583984640 weight_to_token_ratio=0.374941 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205090191e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=917400 combine_payload_bytes=939196416 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940113816 ingress_bytes=1115357184 link_bytes=1115357184 token_bytes=1879310232 weight_movement_bytes=176160768 total_nvlink_bytes=2055471000 weight_to_token_ratio=0.093737 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245534657e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130920 payload_rows_remote=114538 dup_rows=131224 dup_rows_remote=105620 dedup_ratio=0.500580 router_tokens_per_expert=[3, 0, 6, 24, 0, 4, 4585, 0, 7, 8186, 6, 5, 6, 47, 6, 0, 8180, 8185, 8186, 6, 0, 8184, 0, 0, 0, 8158, 0, 8186, 0, 0, 5, 5, 7, 8187, 8182, 0, 8186, 0, 4, 8185, 0, 0, 0, 3, 5, 1, 3, 11, 0, 5, 6, 0, 3634, 1, 0, 4, 0, 0, 8178, 5, 0, 8116, 8184, 8185] payload_rows_per_dst=[16382, 16384, 16380, 16384, 16384, 16384, 16382, 16240] dup_rows_per_dst=[25604, 16510, 24556, 16324, 16424, 16352, 15452, 2] combine_rows_per_src=[16382, 16384, 16380, 16384, 16384, 16384, 16382, 16240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130942 payload_rows_remote=114558 dup_rows=131202 dup_rows_remote=114836 dedup_ratio=0.500496 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3778, 0, 0, 8192, 0, 0, 0, 63, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 8183, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4661, 0, 0, 0, 0, 0, 8191, 0, 0, 7892, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16254] dup_rows_per_dst=[22408, 16508, 23934, 16366, 16384, 16384, 9964, 9254] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=939656 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940463752 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879987848 weight_movement_bytes=352321536 total_nvlink_bytes=2232309384 weight_to_token_ratio=0.187406 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114690 dedup_ratio=0.500015 router_tokens_per_expert=[22, 0, 8186, 0, 3, 4, 0, 15, 8165, 0, 0, 8176, 0, 3, 5, 8187, 0, 4, 0, 1, 0, 1, 0, 3, 8188, 0, 0, 2, 8187, 8188, 8188, 5, 0, 3, 0, 3, 2, 3, 8189, 4, 8188, 4, 5, 0, 8184, 0, 2, 0, 1, 5, 8184, 7, 5, 8185, 8187, 8186, 8188, 2, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16450, 16314, 16386, 16380, 16392, 16382, 16394, 16378] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114686 dedup_ratio=0.500000 router_tokens_per_expert=[72, 0, 8192, 0, 0, 0, 0, 0, 8120, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 8191, 8192, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16528, 16240, 16384, 16384, 16384, 16386, 16382, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938295296 dispatch_meta_bytes=880632 combine_payload_bytes=938295296 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939175928 ingress_bytes=1290616832 link_bytes=1290616832 token_bytes=1877471224 weight_movement_bytes=352321536 total_nvlink_bytes=2229792760 weight_to_token_ratio=0.187657 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226915529e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938459136 dispatch_meta_bytes=917576 combine_payload_bytes=938459136 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939376712 ingress_bytes=1114619904 link_bytes=1114619904 token_bytes=1877835848 weight_movement_bytes=176160768 total_nvlink_bytes=2053996616 weight_to_token_ratio=0.093811 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.241406197e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=115210 dedup_ratio=0.500008 router_tokens_per_expert=[225, 0, 8192, 2, 0, 0, 0, 0, 7932, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8189, 8192, 0, 0, 37, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8191, 0, 0, 8192, 8192, 8176, 8192, 0, 0, 12, 0, 6, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16838, 15864, 16384, 16378, 16458, 16380, 16352, 16420] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917496 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940408824 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900152 weight_movement_bytes=352321536 total_nvlink_bytes=2232221688 weight_to_token_ratio=0.187415 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114668 dup_rows=131092 dup_rows_remote=114706 dedup_ratio=0.500076 router_tokens_per_expert=[111, 0, 8182, 17, 7, 6, 0, 26, 7974, 4, 0, 8173, 0, 7, 9, 8183, 0, 9, 0, 10, 0, 0, 0, 2, 8185, 0, 3, 8, 8180, 8184, 8184, 10, 0, 3, 0, 0, 0, 2, 8183, 9, 8185, 7, 110, 0, 8183, 5, 0, 0, 8, 9, 8155, 14, 8, 8178, 8183, 8162, 8181, 7, 2, 0, 0, 4, 0, 0] payload_rows_per_dst=[16384, 16364, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[13758, 18892, 16358, 16402, 16172, 16578, 16546, 16386] combine_rows_per_src=[16384, 16364, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917496 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940441592 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965688 weight_movement_bytes=440401920 total_nvlink_bytes=2320367608 weight_to_token_ratio=0.234261 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 1, 1, 1, 1, 1, 3971, 0, 1, 8187, 1, 1, 0, 1, 0, 0, 8188, 8191, 8191, 1, 0, 8191, 0, 2, 0, 8189, 2, 8191, 1, 0, 1, 1, 2, 8189, 8191, 0, 8188, 0, 1, 8191, 0, 2, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 4507, 1, 0, 0, 0, 3, 8191, 1, 0, 7904, 8191, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16140, 16380, 8916, 16386, 16384, 16380, 17208, 23278] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114704 dedup_ratio=0.500000 router_tokens_per_expert=[261, 0, 8192, 1, 0, 0, 0, 0, 7902, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 36, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 8184, 8192, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16908, 15804, 16384, 16384, 16456, 16384, 16368, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=919584 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940427296 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879935008 weight_movement_bytes=352321536 total_nvlink_bytes=2232256544 weight_to_token_ratio=0.187412 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 6, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 3] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=917496 combine_payload_bytes=939360256 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940277752 ingress_bytes=1203601408 link_bytes=1203601408 token_bytes=1879638008 weight_movement_bytes=264241152 total_nvlink_bytes=2143879160 weight_to_token_ratio=0.140581 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739665477e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114426 dedup_ratio=0.500015 router_tokens_per_expert=[134, 5, 8184, 0, 6, 2, 0, 1, 8029, 0, 0, 8176, 0, 6, 7, 8184, 0, 6, 0, 6, 0, 2, 2, 5, 8183, 0, 5, 3, 8185, 8187, 8185, 8, 0, 13, 0, 0, 0, 4, 8187, 6, 8186, 6, 19, 0, 8186, 0, 11, 0, 1, 7, 8185, 7, 7, 8182, 8183, 8178, 8186, 5, 1, 0, 0, 0, 0, 1] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16650, 16346, 16402, 16384, 16406, 16200, 16346, 16342] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940441600 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965696 weight_movement_bytes=528482304 total_nvlink_bytes=2408448000 weight_to_token_ratio=0.281113 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[91, 0, 8192, 2, 0, 0, 0, 0, 8103, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 8182, 8192, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16570, 16206, 16384, 16384, 16396, 16384, 16364, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917568 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940441664 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965760 weight_movement_bytes=704643072 total_nvlink_bytes=2584608832 weight_to_token_ratio=0.374817 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114616 dedup_ratio=0.500031 router_tokens_per_expert=[0, 8187, 0, 8168, 5, 0, 0, 0, 8183, 7, 0, 4, 0, 8188, 8188, 0, 8187, 3, 4, 5, 5, 8185, 3, 32, 8184, 3, 4, 8183, 2, 4, 50, 8138, 3, 1, 5, 8188, 4, 4, 1447, 8186, 4, 0, 1, 1, 0, 0, 0, 0, 8187, 1, 0, 5, 5, 0, 8188, 6713, 0, 0, 2, 8187, 18, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16380, 16384, 16382] dup_rows_per_dst=[16338, 16380, 16464, 16386, 19292, 16374, 13438, 16408] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16380, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 4, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=916448 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940424160 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879931872 weight_movement_bytes=176160768 total_nvlink_bytes=2056092640 weight_to_token_ratio=0.093706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 2, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 8158, 0, 5, 31, 8179, 0, 0, 0, 8192, 0, 0, 1330, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 6872, 0, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16388, 16384, 16364, 19044, 16380, 13744, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=940441600 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965696 weight_movement_bytes=528482304 total_nvlink_bytes=2408448000 weight_to_token_ratio=0.281113 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114662 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8191, 14, 8192, 0, 0, 0, 0, 8192, 0, 0, 8168, 0, 0, 8192, 0, 89, 149, 8024, 0, 0, 0, 8192, 0, 0, 992, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 7141, 0, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16410, 16384, 16524, 18368, 16354, 14282, 16366] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=12 unused_prefetch_slots=20 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=6 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 6, 0, 6, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917184 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940375744 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879834304 weight_movement_bytes=352321536 total_nvlink_bytes=2232155840 weight_to_token_ratio=0.187422 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 3, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130962 payload_rows_remote=114600 dup_rows=131182 dup_rows_remote=114696 dedup_ratio=0.500420 router_tokens_per_expert=[0, 8183, 13, 8178, 14, 36, 2, 0, 8181, 1, 0, 8, 0, 8184, 8178, 8, 8180, 3, 8, 9, 1, 8179, 7, 11, 8178, 4, 9, 8153, 7, 49, 86, 8133, 0, 4, 8, 8185, 6, 4, 2521, 8180, 9, 0, 8, 8, 0, 0, 0, 0, 8182, 10, 0, 9, 9, 0, 8184, 5562, 0, 0, 0, 8177, 3, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16298, 16384, 16384, 16384, 16362] dup_rows_per_dst=[16280, 16368, 16314, 16134, 5470, 16466, 27664, 16486] combine_rows_per_src=[16384, 16382, 16384, 16298, 16384, 16384, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=940441616 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965712 weight_movement_bytes=528482304 total_nvlink_bytes=2408448016 weight_to_token_ratio=0.281113 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114672 dedup_ratio=0.500031 router_tokens_per_expert=[489, 0, 8191, 2, 1, 1, 0, 3, 7716, 1, 0, 8187, 0, 0, 1, 8191, 0, 1, 0, 1, 0, 3, 0, 0, 8191, 0, 0, 0, 8191, 8188, 8191, 1, 0, 15, 0, 0, 0, 0, 8189, 1, 8191, 0, 1, 0, 8185, 4, 0, 0, 1, 1, 8191, 0, 1, 8191, 8191, 8159, 8187, 1, 1, 6, 0, 5, 1, 0] payload_rows_per_dst=[16384, 16376, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17372, 15434, 16390, 16376, 16408, 16378, 16322, 16400] combine_rows_per_src=[16384, 16376, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917400 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940441496 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879965592 weight_movement_bytes=616562688 total_nvlink_bytes=2496528280 weight_to_token_ratio=0.327965 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114896 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 8155, 0, 31, 2, 8181, 0, 0, 0, 8192, 0, 0, 974, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 7234, 0, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16356, 16624, 16382, 16176, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114560 dedup_ratio=0.500061 router_tokens_per_expert=[0, 8185, 13, 8174, 8, 44, 13, 0, 8182, 5, 0, 6, 0, 8185, 8187, 0, 8186, 2, 6, 7, 9, 8180, 5, 0, 8184, 1, 7, 8180, 6, 10, 58, 8084, 0, 3, 8, 8186, 4, 1, 968, 8183, 0, 2, 0, 1, 0, 4, 0, 4, 8183, 7, 0, 7, 9, 0, 8186, 7206, 0, 0, 0, 8183, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16370] dup_rows_per_dst=[16528, 16376, 16372, 16312, 18318, 16376, 14440, 16366] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938803200 dispatch_meta_bytes=917184 combine_payload_bytes=938803200 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939720384 ingress_bytes=1291124736 link_bytes=1291124736 token_bytes=1878523584 weight_movement_bytes=352321536 total_nvlink_bytes=2230845120 weight_to_token_ratio=0.187552 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229759580e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917408 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940375968 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834528 weight_movement_bytes=176160768 total_nvlink_bytes=2055995296 weight_to_token_ratio=0.093711 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114602 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8177, 0, 0, 8192, 0, 32, 116, 8087, 0, 0, 0, 8192, 0, 0, 341, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 7824, 0, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16382, 16384, 16470, 17066, 16382, 15648, 16356] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114670 dup_rows=131092 dup_rows_remote=114576 dedup_ratio=0.500076 router_tokens_per_expert=[0, 8187, 8187, 5, 4, 0, 9, 8190, 0, 8181, 0, 8187, 8105, 1, 0, 0, 69, 8186, 8187, 8188, 0, 6, 8186, 8187, 0, 0, 0, 0, 1, 8181, 2, 7, 4, 5, 0, 0, 0, 8, 7, 4, 5, 24, 8179, 5, 2, 8188, 8154, 75, 0, 0, 1, 5, 0, 8127, 3, 0, 0, 5, 5, 0, 6, 4, 0, 0] payload_rows_per_dst=[16384, 16368, 16382, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16400, 16206, 16516, 16380, 16418, 16504, 16264, 16404] combine_rows_per_src=[16384, 16368, 16382, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918336 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940442432 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879966528 weight_movement_bytes=352321536 total_nvlink_bytes=2232288064 weight_to_token_ratio=0.187408 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 3, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=916928 combine_payload_bytes=939393024 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940309952 ingress_bytes=1291714560 link_bytes=1291714560 token_bytes=1879702976 weight_movement_bytes=352321536 total_nvlink_bytes=2232024512 weight_to_token_ratio=0.187435 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233062348e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 8189, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 7934, 0, 0, 0, 252, 8192, 8192, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 2, 15, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16378, 15868, 16888, 16384, 16386, 16388, 16396, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917160 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940441256 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879965352 weight_movement_bytes=616562688 total_nvlink_bytes=2496528040 weight_to_token_ratio=0.327965 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114736 dedup_ratio=0.500031 router_tokens_per_expert=[0, 8192, 8139, 0, 0, 0, 0, 8188, 0, 8192, 0, 8192, 8172, 0, 0, 0, 11, 8192, 8192, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 3, 7, 5, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16278, 16344, 16406, 16374, 16490, 16390, 16408, 16390] combine_rows_per_src=[16376, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=916984 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940293624 ingress_bytes=1820180480 link_bytes=1820180480 token_bytes=1879670264 weight_movement_bytes=880803840 total_nvlink_bytes=2760474104 weight_to_token_ratio=0.468595 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.024621 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019225091e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114670 dedup_ratio=0.500084 router_tokens_per_expert=[0, 8181, 8147, 9, 8, 2, 14, 8181, 0, 8180, 0, 8178, 8147, 4, 0, 0, 6, 8181, 8179, 8184, 0, 33, 8182, 8183, 0, 1, 0, 0, 5, 8172, 3, 25, 9, 8, 0, 0, 0, 34, 8, 9, 10, 21, 8165, 24, 2, 8184, 8160, 6, 130, 2, 0, 0, 5, 8044, 7, 0, 0, 9, 10, 0, 9, 11, 0, 0] payload_rows_per_dst=[16384, 16364, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16312, 16264, 16418, 16416, 16480, 16220, 16560, 16424] combine_rows_per_src=[16384, 16364, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940441584 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879965680 weight_movement_bytes=616562688 total_nvlink_bytes=2496528368 weight_to_token_ratio=0.327965 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917664 combine_payload_bytes=939458560 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940376224 ingress_bytes=1379860480 link_bytes=1379860480 token_bytes=1879834784 weight_movement_bytes=440401920 total_nvlink_bytes=2320236704 weight_to_token_ratio=0.234277 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726642706e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114186 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8191, 3, 8191, 0, 1, 2, 0, 8188, 2, 0, 1, 0, 8184, 8184, 8, 8191, 0, 0, 1, 1, 8191, 1, 0, 8181, 1, 1, 8190, 1, 23, 166, 8115, 0, 1, 1, 8191, 1, 0, 252, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 1, 0, 1, 0, 1, 8191, 7838, 0, 0, 0, 8191, 1, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[16392, 16386, 16386, 16612, 16890, 16368, 15680, 16362] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 8164, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 8164, 0, 0, 0, 28, 8192, 8192, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 8192, 0, 0, 8192, 8191, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16328, 16328, 16440, 16384, 16438, 16382, 16388, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114328 dedup_ratio=0.500053 router_tokens_per_expert=[0, 8184, 8166, 7, 5, 1, 11, 8188, 0, 8183, 0, 8183, 8151, 4, 0, 0, 22, 8181, 8181, 8186, 0, 9, 8185, 8185, 0, 0, 0, 0, 1, 8183, 5, 17, 0, 6, 0, 0, 0, 11, 3, 6, 7, 0, 8174, 12, 0, 8186, 8164, 75, 7, 12, 0, 8, 0, 8124, 4, 0, 0, 7, 7, 1, 8, 6, 6, 0] payload_rows_per_dst=[16384, 16376, 16380, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16758, 18150, 14958, 16398, 15970, 16246, 16298, 16308] combine_rows_per_src=[16384, 16376, 16380, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917344 combine_payload_bytes=939343872 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940261216 ingress_bytes=1291665408 link_bytes=1291665408 token_bytes=1879605088 weight_movement_bytes=352321536 total_nvlink_bytes=2231926624 weight_to_token_ratio=0.187444 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232787117e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114698 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8192, 8151, 0, 0, 0, 0, 8190, 0, 8192, 0, 8192, 7913, 0, 0, 0, 224, 8192, 8192, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 49, 36, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16302, 15826, 16832, 16378, 16450, 16482, 16422, 16384] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=915480 combine_payload_bytes=939491328 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940406808 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879898136 weight_movement_bytes=352321536 total_nvlink_bytes=2232219672 weight_to_token_ratio=0.187415 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 3, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114670 dup_rows=131096 dup_rows_remote=113784 dedup_ratio=0.500092 router_tokens_per_expert=[8147, 0, 0, 1, 5, 0, 0, 0, 1, 8186, 8186, 4, 2, 0, 3, 23, 0, 8186, 10, 0, 474, 8167, 8, 0, 0, 8186, 0, 3, 4, 7, 0, 8187, 3, 5, 8184, 8188, 4, 8187, 0, 0, 1199, 0, 0, 0, 0, 8188, 5, 7704, 5, 6, 8181, 6, 1, 0, 5, 6, 0, 5, 6909, 8180, 8185, 0, 123, 3] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16384, 16384, 16382, 16368] dup_rows_per_dst=[16298, 16426, 17312, 16390, 16382, 17808, 16408, 14072] combine_rows_per_src=[16384, 16384, 16378, 16384, 16384, 16384, 16382, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441584 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965680 weight_movement_bytes=352321536 total_nvlink_bytes=2232287216 weight_to_token_ratio=0.187408 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131000 payload_rows_remote=114688 dup_rows=131144 dup_rows_remote=125896 dedup_ratio=0.500275 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 2041, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 2798, 0, 0, 0, 0, 8192, 0, 7982, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 3556, 8190, 8192, 0, 6, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16312, 16384, 16384] dup_rows_per_dst=[16384, 16384, 20468, 16384, 16384, 5248, 16388, 23504] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16312, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131002 payload_rows_remote=114618 dup_rows=131142 dup_rows_remote=114758 dedup_ratio=0.500267 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 40, 0, 1738, 8192, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 2199, 0, 0, 0, 0, 8191, 0, 7015, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5423, 8155, 8192, 0, 5, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16314] dup_rows_per_dst=[16384, 16384, 19944, 16384, 16384, 18426, 16384, 10852] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=916008 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940325416 ingress_bytes=1379811328 link_bytes=1379811328 token_bytes=1879734824 weight_movement_bytes=440401920 total_nvlink_bytes=2320136744 weight_to_token_ratio=0.234289 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726367475e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917528 combine_payload_bytes=939491328 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940408856 ingress_bytes=1203732480 link_bytes=1203732480 token_bytes=1879900184 weight_movement_bytes=264241152 total_nvlink_bytes=2144141336 weight_to_token_ratio=0.140561 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740399425e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=913816 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940290456 ingress_bytes=1379778560 link_bytes=1379778560 token_bytes=1879667096 weight_movement_bytes=440401920 total_nvlink_bytes=2320069016 weight_to_token_ratio=0.234298 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726183988e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130910 payload_rows_remote=114534 dup_rows=131234 dup_rows_remote=108430 dedup_ratio=0.500618 router_tokens_per_expert=[8162, 0, 0, 16, 10, 0, 4, 0, 0, 8177, 8182, 8, 0, 0, 11, 6, 0, 8181, 15, 1, 1058, 8113, 20, 5, 4, 8181, 2, 7, 9, 11, 0, 8181, 0, 11, 8176, 8183, 5, 8181, 0, 9, 4149, 0, 0, 0, 0, 8180, 0, 8059, 12, 11, 8174, 14, 4, 0, 4, 10, 0, 9, 3077, 8170, 8179, 0, 84, 7] payload_rows_per_dst=[16384, 16384, 16242, 16384, 16384, 16372, 16384, 16376] dup_rows_per_dst=[16366, 16282, 2992, 16408, 16518, 23558, 16306, 22804] combine_rows_per_src=[16384, 16384, 16242, 16384, 16384, 16372, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=962336 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940486432 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1880010528 weight_movement_bytes=352321536 total_nvlink_bytes=2232332064 weight_to_token_ratio=0.187404 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938950656 dispatch_meta_bytes=917504 combine_payload_bytes=938950656 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=939868160 ingress_bytes=1203191808 link_bytes=1203191808 token_bytes=1878818816 weight_movement_bytes=264241152 total_nvlink_bytes=2143059968 weight_to_token_ratio=0.140642 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.737371888e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114638 dedup_ratio=0.500031 router_tokens_per_expert=[0, 8191, 8167, 1, 1, 0, 0, 8187, 1, 8191, 0, 8191, 8064, 1, 0, 0, 122, 8191, 8191, 8188, 0, 1, 8191, 8191, 0, 0, 0, 0, 0, 8179, 1, 1, 0, 0, 0, 1, 0, 26, 3, 1, 1, 0, 8190, 2, 1, 8191, 8187, 41, 8, 5, 0, 1, 0, 8155, 1, 0, 0, 1, 1, 1, 1, 1, 2, 0] payload_rows_per_dst=[16376, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16336, 16130, 16620, 16360, 16442, 16460, 16338, 16394] combine_rows_per_src=[16376, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114692 dedup_ratio=0.500015 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 22, 0, 682, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 2510, 0, 0, 0, 0, 8192, 0, 8060, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5122, 8169, 8192, 0, 11, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[16384, 16384, 17256, 16384, 16384, 5296, 16384, 26604] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938262528 dispatch_meta_bytes=891856 combine_payload_bytes=938262528 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939154384 ingress_bytes=1290584064 link_bytes=1290584064 token_bytes=1877416912 weight_movement_bytes=352321536 total_nvlink_bytes=2229738448 weight_to_token_ratio=0.187663 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226732042e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114644 dup_rows=131116 dup_rows_remote=114626 dedup_ratio=0.500168 router_tokens_per_expert=[8182, 1, 0, 1, 8, 0, 1, 0, 0, 8184, 8182, 6, 0, 0, 8, 19, 6, 8184, 18, 1, 988, 8142, 4, 0, 4, 8182, 0, 4, 7, 10, 0, 8184, 0, 8, 8181, 8185, 4, 8184, 0, 0, 2935, 5, 0, 0, 2, 8181, 6, 8095, 9, 9, 8165, 9, 8, 0, 4, 8, 0, 7, 4322, 8175, 8184, 0, 55, 5] payload_rows_per_dst=[16384, 16384, 16376, 16376, 16378, 16382, 16384, 16364] dup_rows_per_dst=[16490, 16430, 18340, 16360, 16258, 22066, 16408, 8764] combine_rows_per_src=[16384, 16384, 16376, 16376, 16378, 16382, 16384, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917272 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940375832 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879834392 weight_movement_bytes=352321536 total_nvlink_bytes=2232155928 weight_to_token_ratio=0.187422 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131012 payload_rows_remote=114628 dup_rows=131132 dup_rows_remote=114748 dedup_ratio=0.500229 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 1, 0, 8192, 36, 0, 347, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 4360, 0, 0, 0, 0, 8192, 0, 7832, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3775, 8162, 8192, 0, 63, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16324] dup_rows_per_dst=[16384, 16386, 17150, 16384, 16384, 24260, 16384, 7800] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114680 dedup_ratio=0.500046 router_tokens_per_expert=[3, 0, 8188, 6, 5, 0, 5, 0, 0, 8186, 5, 2, 8186, 9, 8182, 8185, 3, 0, 2, 8187, 4, 6, 0, 6, 0, 6, 0, 0, 0, 1, 4, 4, 0, 8187, 8187, 0, 8167, 5, 0, 0, 2, 7, 3, 0, 5, 8072, 3, 5, 0, 0, 434, 0, 8022, 6, 1, 0, 0, 8142, 8187, 8082, 0, 8186, 8184, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16372] dup_rows_per_dst=[16404, 16382, 16404, 16392, 16334, 16182, 16910, 16076] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917504 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940408832 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879900160 weight_movement_bytes=176160768 total_nvlink_bytes=2056060928 weight_to_token_ratio=0.093708 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114720 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 663, 0, 7858, 0, 0, 0, 0, 8192, 8192, 7879, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16352, 17042, 15758] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 518, 1, 8162, 0, 0, 0, 0, 8188, 8192, 7729, 7, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16328, 17362, 15464] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=917080 combine_payload_bytes=939163648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940080728 ingress_bytes=1291485184 link_bytes=1291485184 token_bytes=1879244376 weight_movement_bytes=352321536 total_nvlink_bytes=2231565912 weight_to_token_ratio=0.187480 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231777938e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=917504 combine_payload_bytes=939032576 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=939950080 ingress_bytes=1291354112 link_bytes=1291354112 token_bytes=1878982656 weight_movement_bytes=352321536 total_nvlink_bytes=2231304192 weight_to_token_ratio=0.187507 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231043989e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917424 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940343216 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879769008 weight_movement_bytes=176160768 total_nvlink_bytes=2055929776 weight_to_token_ratio=0.093714 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131006 payload_rows_remote=114672 dup_rows=131138 dup_rows_remote=110814 dedup_ratio=0.500252 router_tokens_per_expert=[7, 0, 8182, 10, 7, 0, 12, 0, 0, 8180, 9, 0, 8180, 14, 8180, 8178, 12, 0, 6, 8181, 10, 11, 0, 14, 14, 12, 0, 0, 0, 1, 8, 0, 0, 8178, 8181, 0, 8164, 8, 0, 0, 13, 6, 23, 0, 11, 7874, 3, 12, 0, 1, 571, 0, 8160, 6, 10, 0, 0, 8152, 8182, 7930, 3, 8181, 8175, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16372, 16384, 16380, 16334] dup_rows_per_dst=[16338, 15160, 16444, 16350, 16404, 17078, 13040, 20324] combine_rows_per_src=[16384, 16384, 16384, 16384, 16372, 16384, 16380, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917632 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441728 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965824 weight_movement_bytes=352321536 total_nvlink_bytes=2232287360 weight_to_token_ratio=0.187408 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017045 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940441600 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965696 weight_movement_bytes=704643072 total_nvlink_bytes=2584608768 weight_to_token_ratio=0.374817 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 4, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114710 dedup_ratio=0.500084 router_tokens_per_expert=[8191, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 1, 7, 0, 8191, 24, 1, 672, 8181, 0, 6, 2, 8190, 0, 0, 1, 1, 0, 8191, 0, 1, 8191, 8191, 1, 8191, 0, 0, 5443, 1, 0, 1, 0, 8191, 1, 7892, 0, 1, 8189, 1, 1, 0, 14, 1, 1, 1, 2273, 8182, 8191, 0, 70, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16366] dup_rows_per_dst=[16382, 16396, 17766, 16386, 16384, 14504, 16412, 16864] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114338 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 214, 0, 8153, 0, 0, 0, 0, 8185, 8192, 8032, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16368, 16734, 16050] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=901944 combine_payload_bytes=939393024 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=940294968 ingress_bytes=1820196864 link_bytes=1820196864 token_bytes=1879687992 weight_movement_bytes=880803840 total_nvlink_bytes=2760491832 weight_to_token_ratio=0.468590 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019234265e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131006 payload_rows_remote=114622 dup_rows=131138 dup_rows_remote=114732 dedup_ratio=0.500252 router_tokens_per_expert=[4, 0, 8184, 6, 8, 0, 9, 0, 0, 8182, 7, 0, 8183, 16, 8178, 8181, 7, 7, 4, 8184, 6, 0, 0, 10, 5, 10, 0, 0, 0, 0, 6, 6, 0, 8182, 8184, 0, 8173, 4, 0, 0, 1, 13, 13, 0, 9, 8123, 4, 10, 0, 3, 105, 0, 8145, 7, 0, 0, 7, 8180, 8185, 8158, 0, 8184, 8179, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16372, 16384, 16384, 16330] dup_rows_per_dst=[16406, 17618, 16294, 16408, 16454, 16328, 16494, 15136] combine_rows_per_src=[16384, 16384, 16384, 16384, 16372, 16384, 16384, 16330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917504 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940261376 ingress_bytes=1467826176 link_bytes=1467826176 token_bytes=1879605248 weight_movement_bytes=528482304 total_nvlink_bytes=2408087552 weight_to_token_ratio=0.281167 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219213885e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 60, 0, 8183, 0, 0, 0, 0, 8192, 8192, 8152, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16362, 16486, 16304] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916104 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940440200 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879964296 weight_movement_bytes=176160768 total_nvlink_bytes=2056125064 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 1, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114678 dup_rows=131084 dup_rows_remote=114696 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 4, 8151, 6, 8186, 0, 8185, 0, 7, 0, 19, 0, 0, 8183, 0, 23, 8184, 4, 8171, 3, 8185, 8187, 0, 8179, 6, 8172, 0, 0, 5, 2, 5, 0, 7, 4, 9, 7, 0, 0, 0, 0, 0, 4, 5, 7, 0, 8182, 54, 8186, 0, 49, 0, 5, 1, 8187, 0, 6, 8188, 0, 4, 5, 8184, 0, 8111] payload_rows_per_dst=[16384, 16384, 16382, 16374, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16308, 16406, 16388, 16364, 16414, 16488, 16472, 16244] combine_rows_per_src=[16384, 16384, 16382, 16374, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114700 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 8192, 0, 32, 0, 0, 0, 8192, 0, 0, 8181, 0, 0, 0, 8192, 0, 8165] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16372] dup_rows_per_dst=[16384, 16384, 16384, 16384, 16390, 16390, 16448, 16320] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=917416 combine_payload_bytes=938983424 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939900840 ingress_bytes=1115144192 link_bytes=1115144192 token_bytes=1878884264 weight_movement_bytes=176160768 total_nvlink_bytes=2055045032 weight_to_token_ratio=0.093758 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244341991e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114638 dup_rows=131122 dup_rows_remote=114224 dedup_ratio=0.500191 router_tokens_per_expert=[0, 0, 7, 8154, 10, 8177, 0, 8174, 0, 16, 0, 47, 0, 0, 8176, 0, 21, 8146, 10, 8120, 0, 8178, 8181, 0, 8168, 8, 8150, 0, 0, 9, 57, 10, 1, 13, 6, 16, 14, 0, 2, 0, 0, 4, 8, 11, 15, 0, 8168, 38, 8176, 0, 248, 4, 9, 2, 8183, 18, 12, 8171, 0, 9, 9, 8175, 0, 7941] payload_rows_per_dst=[16382, 16384, 16342, 16378, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16308, 16448, 16248, 16404, 16438, 16134, 16244, 16898] combine_rows_per_src=[16382, 16384, 16342, 16378, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917496 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940359672 ingress_bytes=1467924480 link_bytes=1467924480 token_bytes=1879801848 weight_movement_bytes=528482304 total_nvlink_bytes=2408284152 weight_to_token_ratio=0.281137 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219764346e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917504 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940343296 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879769088 weight_movement_bytes=176160768 total_nvlink_bytes=2055929856 weight_to_token_ratio=0.093714 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114684 dup_rows=131080 dup_rows_remote=114706 dedup_ratio=0.500031 router_tokens_per_expert=[1, 0, 8191, 1, 1, 0, 1, 0, 0, 8190, 1, 1, 8191, 2, 8185, 8191, 1, 0, 1, 8191, 1, 1, 3, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 8188, 8191, 0, 8188, 1, 0, 0, 0, 2, 0, 1, 1, 8033, 0, 1, 0, 0, 397, 0, 8107, 0, 2, 0, 0, 8190, 8191, 8036, 4, 8191, 8190, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16382, 16382] dup_rows_per_dst=[16388, 16376, 16394, 16386, 16374, 16074, 17010, 16078] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114678 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 8187, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 5, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16374, 16384, 16384, 16394, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=915448 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940029944 ingress_bytes=1291436032 link_bytes=1291436032 token_bytes=1879144440 weight_movement_bytes=352321536 total_nvlink_bytes=2231465976 weight_to_token_ratio=0.187490 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231502707e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114536 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 6, 8171, 9, 8182, 0, 8179, 0, 10, 0, 39, 0, 0, 8180, 0, 17, 8171, 12, 8152, 0, 8182, 8182, 0, 8140, 7, 8168, 0, 0, 7, 42, 8, 0, 8, 4, 17, 11, 0, 0, 0, 0, 0, 6, 9, 10, 5, 8178, 66, 8168, 0, 69, 0, 8, 1, 8184, 12, 10, 8185, 0, 0, 11, 8181, 0, 8065] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16538, 16292, 16570, 16412, 16268, 16526, 16310, 16158] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917560 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940408888 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900216 weight_movement_bytes=352321536 total_nvlink_bytes=2232221752 weight_to_token_ratio=0.187415 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114696 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 8192, 8192, 0, 8189, 0, 8192, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 14, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 8192, 0, 8178] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16384, 16384, 16384, 16378, 16398, 16384, 16410, 16352] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114682 dup_rows=131092 dup_rows_remote=115142 dedup_ratio=0.500076 router_tokens_per_expert=[1, 0, 8187, 0, 0, 2, 3, 8186, 5, 0, 3, 5, 2, 5, 0, 3386, 0, 0, 7975, 1, 8183, 8185, 0, 8184, 0, 0, 20, 6, 8187, 4, 8186, 0, 43, 7, 8145, 9, 2, 0, 0, 5, 0, 3, 5008, 0, 8186, 7, 2, 8186, 5, 8187, 8187, 0, 0, 8178, 0, 6, 0, 6, 7, 0, 3, 8169, 0, 5] payload_rows_per_dst=[16378, 16384, 16370, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 23164, 15950, 16422, 16410, 10028, 16368, 16370] combine_rows_per_src=[16378, 16384, 16370, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917464 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940441560 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879965656 weight_movement_bytes=616562688 total_nvlink_bytes=2496528344 weight_to_token_ratio=0.327965 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.018939 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=109136 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2592, 0, 0, 8190, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5602, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 8484, 18732, 16384, 16384, 21936, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=110894 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 1897, 5, 0, 8191, 0, 8192, 8192, 1, 8192, 0, 0, 0, 0, 8180, 0, 8192, 1, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 6290, 0, 8192, 0, 0, 8192, 0, 8192, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16420, 20178, 16394, 16362, 16386, 12580, 16380, 16372] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=916888 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940424600 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932312 weight_movement_bytes=352321536 total_nvlink_bytes=2232253848 weight_to_token_ratio=0.187412 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 4, 0, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917528 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940425240 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932952 weight_movement_bytes=264241152 total_nvlink_bytes=2144174104 weight_to_token_ratio=0.140559 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131036 payload_rows_remote=114652 dup_rows=131108 dup_rows_remote=114704 dedup_ratio=0.500137 router_tokens_per_expert=[0, 4, 8181, 0, 0, 5, 10, 8171, 12, 0, 5, 11, 6, 7, 0, 2327, 0, 0, 7668, 5, 8175, 8179, 0, 8174, 0, 0, 32, 11, 8168, 0, 8180, 1, 61, 17, 8111, 17, 15, 0, 8, 0, 0, 12, 6374, 0, 8178, 15, 0, 8177, 11, 8183, 8181, 0, 0, 8170, 0, 10, 0, 10, 13, 0, 7, 8158, 0, 12] payload_rows_per_dst=[16372, 16384, 16360, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16368, 4710, 15294, 16370, 16326, 29224, 16412, 16404] combine_rows_per_src=[16372, 16384, 16360, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=919296 combine_payload_bytes=939474944 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=940394240 ingress_bytes=1820278784 link_bytes=1820278784 token_bytes=1879869184 weight_movement_bytes=880803840 total_nvlink_bytes=2760673024 weight_to_token_ratio=0.468545 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019280137e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=895296 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940419392 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879943488 weight_movement_bytes=352321536 total_nvlink_bytes=2232265024 weight_to_token_ratio=0.187411 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=902328 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940426424 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879950520 weight_movement_bytes=352321536 total_nvlink_bytes=2232272056 weight_to_token_ratio=0.187410 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114678 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 1, 8190, 1, 8190, 1, 8191, 0, 1, 0, 5, 0, 0, 8191, 0, 0, 8187, 0, 8191, 1, 8191, 8191, 0, 8189, 1, 8190, 2, 3, 1, 0, 1, 5, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 8190, 0, 8186, 0, 20, 0, 1, 2, 8188, 0, 1, 8181, 0, 0, 2, 8190, 0, 8181] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16384, 16390, 16374, 16390, 16396, 16384, 16410, 16346] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=917424 combine_payload_bytes=939229184 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940146608 ingress_bytes=1291550720 link_bytes=1291550720 token_bytes=1879375792 weight_movement_bytes=352321536 total_nvlink_bytes=2231697328 weight_to_token_ratio=0.187467 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232144912e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 0, 8186, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4869, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 6658, 16372, 16382, 16384, 26122, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114678 dup_rows=131092 dup_rows_remote=114716 dedup_ratio=0.500076 router_tokens_per_expert=[0, 2, 8180, 0, 0, 5, 1, 8182, 9, 2, 4, 8, 4, 6, 0, 4550, 0, 0, 7839, 4, 8179, 8178, 0, 8178, 0, 0, 28, 9, 8173, 0, 8183, 0, 45, 12, 8137, 6, 15, 0, 9, 0, 0, 6, 3972, 0, 8181, 12, 2, 8183, 8, 8185, 8183, 0, 0, 8181, 0, 10, 0, 9, 6, 5, 6, 8166, 0, 9] payload_rows_per_dst=[16374, 16384, 16374, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16376, 25496, 15676, 16384, 16430, 7962, 16380, 16388] combine_rows_per_src=[16374, 16384, 16374, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114690 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 3207, 0, 0, 8190, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4985, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16394, 22798, 16380, 16382, 16384, 9970, 16384, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114678 dup_rows=131088 dup_rows_remote=114612 dedup_ratio=0.500061 router_tokens_per_expert=[6, 7, 4, 0, 4, 8159, 0, 0, 8, 2, 2, 0, 0, 5, 0, 8185, 0, 8182, 4, 8180, 1, 8183, 8183, 57, 8187, 2, 8186, 6, 0, 1669, 8184, 8183, 0, 2, 0, 33, 0, 8181, 0, 8185, 6, 8185, 5, 0, 8184, 5, 4, 5, 6, 0, 0, 4, 1, 6, 8186, 0, 0, 6397, 0, 12, 65, 0, 6, 5] payload_rows_per_dst=[16384, 16384, 16378, 16374, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16342, 16392, 16476, 19714, 16418, 16404, 16382, 12960] combine_rows_per_src=[16384, 16384, 16378, 16374, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917456 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940425168 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932880 weight_movement_bytes=352321536 total_nvlink_bytes=2232254416 weight_to_token_ratio=0.187412 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 8192, 0, 8192, 8192, 4, 8192, 0, 8192, 0, 0, 2270, 8192, 8192, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5077, 0, 0, 847, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16378, 16384, 16388, 8780, 16382, 16384, 16384, 23992] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441600 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965696 weight_movement_bytes=528482304 total_nvlink_bytes=2408448000 weight_to_token_ratio=0.281113 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 7994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8185, 0, 8192, 0, 8189, 8192, 12, 8192, 0, 8192, 29, 0, 5260, 8192, 8182, 0, 0, 0, 0, 0, 8112, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3102, 0, 0, 79, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15988, 16384, 16388, 26942, 16224, 16400, 16384, 6362] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917576 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940359752 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879801928 weight_movement_bytes=176160768 total_nvlink_bytes=2055962696 weight_to_token_ratio=0.093712 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130864 payload_rows_remote=114480 dup_rows=131280 dup_rows_remote=105356 dedup_ratio=0.500793 router_tokens_per_expert=[13, 15, 8, 0, 7, 8115, 0, 0, 17, 3, 2, 0, 2, 11, 0, 8178, 0, 8171, 7, 8171, 4, 8176, 8175, 64, 8181, 5, 8182, 29, 0, 3400, 8172, 8176, 0, 6, 0, 62, 0, 8147, 0, 8177, 11, 8166, 10, 0, 8172, 11, 10, 8, 12, 0, 0, 6, 0, 14, 8182, 0, 0, 4619, 0, 9, 143, 0, 12, 11] payload_rows_per_dst=[16384, 16384, 16376, 16192, 16384, 16376, 16384, 16384] dup_rows_per_dst=[16292, 16366, 16450, 6646, 16808, 16400, 16394, 25924] combine_rows_per_src=[16384, 16384, 16376, 16192, 16384, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917512 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940441608 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965704 weight_movement_bytes=352321536 total_nvlink_bytes=2232287240 weight_to_token_ratio=0.187408 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917160 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940359336 ingress_bytes=1644085248 link_bytes=1644085248 token_bytes=1879801512 weight_movement_bytes=704643072 total_nvlink_bytes=2584444584 weight_to_token_ratio=0.374850 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206191114e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=264241152 egress_bytes=940441600 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965696 weight_movement_bytes=264241152 total_nvlink_bytes=2144206848 weight_to_token_ratio=0.140556 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937820160 dispatch_meta_bytes=879344 combine_payload_bytes=937820160 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938699504 ingress_bytes=1290141696 link_bytes=1290141696 token_bytes=1876519664 weight_movement_bytes=352321536 total_nvlink_bytes=2228841200 weight_to_token_ratio=0.187753 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224254966e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29939961032 prefetch_bytes=2730491904 reduce_bytes=4139778048 weight_movement_bytes=6870269952 egress_bytes=14977305800 ingress_bytes=21832925184 link_bytes=21832925184 total_nvlink_bytes=36810230984 weight_to_token_ratio=0.229468 prefetch_share_of_ingress=0.125063 dedup_ratio=0.502312 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=47 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020774 gemm_inner_pad_upper_bound=33274 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.640954220e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=30001913352 prefetch_bytes=2994733056 reduce_bytes=3787456512 weight_movement_bytes=6782189568 egress_bytes=15008292360 ingress_bytes=21775810560 link_bytes=21775810560 total_nvlink_bytes=36784102920 weight_to_token_ratio=0.226059 prefetch_share_of_ingress=0.137526 dedup_ratio=0.501304 prefetch_slots=68 unused_prefetch_slots=444 slot_utilization=0.132812 reduce_slots=43 remote_demand=34 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020419 gemm_inner_pad_upper_bound=34290 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.620965592e-05 prefetch_bytes_per_gemm_flop=1.048078441e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29861956040 prefetch_bytes=2730491904 reduce_bytes=2730491904 weight_movement_bytes=5460983808 egress_bytes=14938261960 ingress_bytes=20384677888 link_bytes=20384677888 total_nvlink_bytes=35322939848 weight_to_token_ratio=0.182874 prefetch_share_of_ingress=0.133948 dedup_ratio=0.503211 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=31 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021366 gemm_inner_pad_upper_bound=32258 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.134105449e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114706 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 8188, 0, 0, 0, 10, 8190, 1, 1, 1, 1, 1, 1, 0, 1596, 5, 0, 8147, 1, 8191, 8191, 9, 8191, 0, 0, 2, 1, 8189, 0, 8191, 2, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 1, 6624, 0, 8191, 1, 0, 8191, 1, 8191, 8191, 0, 0, 8189, 0, 1, 0, 1, 2, 0, 1, 8191, 0, 1] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16394, 19584, 16322, 16386, 16370, 13250, 16380, 16390] combine_rows_per_src=[16382, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114638 dup_rows=131122 dup_rows_remote=113842 dedup_ratio=0.500191 router_tokens_per_expert=[9, 11, 5, 0, 6, 8154, 0, 0, 13, 0, 3, 0, 1, 8, 0, 8181, 0, 8158, 5, 8177, 3, 8176, 8180, 208, 8183, 4, 8183, 16, 0, 2338, 8180, 8183, 0, 4, 0, 29, 0, 8174, 0, 8181, 8, 8175, 8, 0, 8178, 9, 7, 6, 9, 0, 0, 5, 5, 10, 8184, 0, 0, 5662, 0, 7, 7, 0, 11, 8] payload_rows_per_dst=[16384, 16384, 16376, 16368, 16382, 16360, 16384, 16384] dup_rows_per_dst=[17280, 16394, 16258, 20722, 16394, 16334, 16368, 11372] combine_rows_per_src=[16384, 16384, 16376, 16368, 16382, 16360, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 8192, 0, 8192, 8192, 1, 8192, 0, 8192, 0, 0, 3453, 8192, 8192, 0, 0, 0, 0, 0, 8149, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4736, 0, 0, 78, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16320, 16384, 16384, 6906, 16298, 16384, 16384, 26012] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29927342456 prefetch_bytes=3170893824 reduce_bytes=3699376128 weight_movement_bytes=6870269952 egress_bytes=14970847608 ingress_bytes=21826764800 link_bytes=21826764800 total_nvlink_bytes=36797612408 weight_to_token_ratio=0.229565 prefetch_share_of_ingress=0.145275 dedup_ratio=0.502234 prefetch_slots=72 unused_prefetch_slots=440 slot_utilization=0.140625 reduce_slots=42 remote_demand=36 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=34036 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.638798246e-05 prefetch_bytes_per_gemm_flop=1.109730114e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=109024 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8172, 0, 8192, 0, 8190, 8192, 11, 8192, 0, 8192, 21, 0, 2820, 8192, 8185, 0, 0, 0, 0, 0, 8139, 0, 8192, 0, 8192, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5444, 0, 0, 80, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16162, 16384, 16366, 22052, 16278, 16402, 16384, 11048] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917568 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425280 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932992 weight_movement_bytes=176160768 total_nvlink_bytes=2056093760 weight_to_token_ratio=0.093706 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=913920 combine_payload_bytes=939114496 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940028416 ingress_bytes=1291436032 link_bytes=1291436032 token_bytes=1879142912 weight_movement_bytes=352321536 total_nvlink_bytes=2231464448 weight_to_token_ratio=0.187491 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231502707e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 2, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=894832 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=792723456 egress_bytes=940386160 ingress_bytes=1908375552 link_bytes=1908375552 token_bytes=1879877488 weight_movement_bytes=968884224 total_nvlink_bytes=2848761712 weight_to_token_ratio=0.515398 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.068610650e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29822437376 prefetch_bytes=3875536896 reduce_bytes=1056964608 weight_movement_bytes=4932501504 egress_bytes=14918502400 ingress_bytes=19836436480 link_bytes=19836436480 total_nvlink_bytes=34754938880 weight_to_token_ratio=0.165396 prefetch_share_of_ingress=0.195375 dedup_ratio=0.503787 prefetch_slots=88 unused_prefetch_slots=424 slot_utilization=0.171875 reduce_slots=12 remote_demand=44 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020360 gemm_inner_pad_upper_bound=34925 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.942235259e-05 prefetch_bytes_per_gemm_flop=1.356336806e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29972800216 prefetch_bytes=3347054592 reduce_bytes=2025848832 weight_movement_bytes=5372903424 egress_bytes=14993744600 ingress_bytes=20351959040 link_bytes=20351959040 total_nvlink_bytes=35345703640 weight_to_token_ratio=0.179259 prefetch_share_of_ingress=0.164459 dedup_ratio=0.501700 prefetch_slots=76 unused_prefetch_slots=436 slot_utilization=0.148438 reduce_slots=23 remote_demand=38 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020241 gemm_inner_pad_upper_bound=34544 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.122654706e-05 prefetch_bytes_per_gemm_flop=1.171381787e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29961041896 prefetch_bytes=2554331136 reduce_bytes=4227858432 weight_movement_bytes=6782189568 egress_bytes=14987835368 ingress_bytes=21755396096 link_bytes=21755396096 total_nvlink_bytes=36743231464 weight_to_token_ratio=0.226367 prefetch_share_of_ingress=0.117411 dedup_ratio=0.501862 prefetch_slots=58 unused_prefetch_slots=454 slot_utilization=0.113281 reduce_slots=48 remote_demand=29 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020893 gemm_inner_pad_upper_bound=33401 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.613821062e-05 prefetch_bytes_per_gemm_flop=8.939492582e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114808 dedup_ratio=0.500031 router_tokens_per_expert=[1, 1, 1, 0, 1, 8154, 0, 0, 0, 1, 1, 0, 0, 1, 1, 8191, 0, 8186, 1, 8191, 0, 8190, 8190, 8, 8191, 0, 8189, 12, 0, 2203, 8190, 8187, 0, 0, 0, 8, 0, 8129, 1, 8190, 1, 8191, 1, 0, 8190, 1, 5, 1, 1, 0, 0, 1, 0, 1, 8191, 3, 0, 6015, 0, 1, 48, 1, 0, 1] payload_rows_per_dst=[16382, 16384, 16382, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16312, 16384, 16390, 19532, 16272, 16396, 16392, 13402] combine_rows_per_src=[16382, 16384, 16382, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917952 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940376512 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879835072 weight_movement_bytes=176160768 total_nvlink_bytes=2055995840 weight_to_token_ratio=0.093711 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.025568 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29992544552 prefetch_bytes=2994733056 reduce_bytes=2730491904 weight_movement_bytes=5725224960 egress_bytes=15003609384 ingress_bytes=20714160128 link_bytes=20714160128 total_nvlink_bytes=35717769512 weight_to_token_ratio=0.190888 prefetch_share_of_ingress=0.144574 dedup_ratio=0.501339 prefetch_slots=68 unused_prefetch_slots=444 slot_utilization=0.132812 reduce_slots=31 remote_demand=34 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021366 gemm_inner_pad_upper_bound=32639 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.249415638e-05 prefetch_bytes_per_gemm_flop=1.048078441e-05 anomalies=none - [2026-09-16 21:39:32] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 622.5 | throughput per GPU (TFLOP/s/GPU): 535.6 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.038050E+01 | loss scale: 1.0 | grad norm: 32.675 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124382 payload_rows_remote=109080 dup_rows=137762 dup_rows_remote=125208 dedup_ratio=0.525520 router_tokens_per_expert=[214, 8, 144, 2, 221, 1, 4214, 3, 3271, 589, 138, 4783, 4198, 5382, 0, 7126, 6356, 190, 270, 3236, 487, 6794, 282, 2669, 3, 247, 274, 4173, 3499, 9, 1404, 1626, 2904, 39, 383, 6397, 5126, 10, 1969, 1553, 1286, 17, 2678, 5322, 4, 585, 3110, 2133, 1678, 7208, 3813, 3, 9, 24, 9, 5762, 0, 3388, 447, 1238, 2739, 3499, 5658, 268] payload_rows_per_dst=[16110, 15988, 15302, 15886, 14876, 14868, 15602, 15750] dup_rows_per_dst=[18520, 22682, 12554, 19378, 13614, 15402, 16888, 18724] combine_rows_per_src=[16110, 15988, 15302, 15886, 14876, 14868, 15602, 15750] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126094 payload_rows_remote=110466 dup_rows=136050 dup_rows_remote=117224 dedup_ratio=0.518990 router_tokens_per_expert=[98, 2, 83, 2, 194, 1, 4506, 5, 1706, 427, 45, 3689, 3572, 4698, 3, 7190, 6998, 266, 269, 3844, 192, 7353, 133, 2739, 7, 242, 383, 3974, 3811, 13, 1032, 1672, 4028, 47, 385, 6249, 3644, 3, 3100, 2304, 1718, 3, 2178, 5172, 2, 431, 4946, 2777, 1335, 7202, 3438, 1, 3, 12, 16, 6413, 0, 2219, 359, 1042, 2954, 4142, 5609, 191] payload_rows_per_dst=[16156, 15852, 15926, 15938, 14556, 15628, 16280, 15758] dup_rows_per_dst=[17402, 17028, 13666, 18828, 12466, 18826, 20560, 17274] combine_rows_per_src=[16156, 15852, 15926, 15938, 14556, 15628, 16280, 15758] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126092 payload_rows_remote=109892 dup_rows=136052 dup_rows_remote=113818 dedup_ratio=0.518997 router_tokens_per_expert=[168, 9, 211, 15, 281, 3, 3612, 1, 1503, 328, 61, 3085, 5170, 4144, 2, 7534, 7060, 244, 419, 5024, 116, 6230, 299, 2347, 0, 89, 406, 4744, 3362, 7, 949, 1156, 3864, 89, 352, 5647, 3663, 6, 3944, 2279, 1663, 21, 2753, 4715, 5, 409, 4230, 3858, 1926, 7188, 4157, 5, 2, 52, 65, 5822, 0, 2516, 206, 705, 3264, 3189, 5674, 224] payload_rows_per_dst=[16218, 15526, 15632, 15644, 15212, 15960, 16200, 15700] dup_rows_per_dst=[15738, 16584, 16034, 16338, 13920, 19348, 22234, 15856] combine_rows_per_src=[16218, 15526, 15632, 15644, 15212, 15960, 16200, 15700] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122510 payload_rows_remote=106520 dup_rows=139634 dup_rows_remote=119284 dedup_ratio=0.532661 router_tokens_per_expert=[89, 1, 126, 9, 106, 2, 4222, 1, 1839, 519, 85, 3368, 3461, 4880, 2, 7621, 7420, 414, 448, 3575, 200, 6892, 237, 2681, 0, 244, 332, 4050, 3352, 14, 1199, 1666, 4097, 24, 473, 6033, 3548, 4, 2533, 1797, 1147, 7, 2790, 4716, 0, 440, 4836, 3577, 1845, 7605, 4074, 2, 0, 38, 37, 5987, 0, 2717, 324, 885, 2973, 3782, 5545, 181] payload_rows_per_dst=[16204, 13142, 16302, 14004, 15320, 15270, 16278, 15990] dup_rows_per_dst=[15270, 8606, 26872, 8272, 21136, 16480, 22648, 20350] combine_rows_per_src=[16204, 13142, 16302, 14004, 15320, 15270, 16278, 15990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125906 payload_rows_remote=110140 dup_rows=136238 dup_rows_remote=119112 dedup_ratio=0.519707 router_tokens_per_expert=[121, 13, 133, 5, 287, 1, 4169, 4, 1895, 447, 85, 3535, 3361, 4710, 3, 7561, 7466, 232, 312, 3248, 366, 6723, 226, 2684, 10, 305, 249, 4352, 3245, 5, 957, 1468, 4170, 21, 331, 5855, 4436, 9, 2754, 2661, 1568, 15, 2662, 4993, 3, 587, 3967, 3591, 1775, 7233, 4031, 0, 8, 3, 40, 5753, 2, 2202, 255, 1029, 2761, 4206, 5776, 197] payload_rows_per_dst=[16204, 16082, 15342, 15766, 14790, 15818, 16232, 15672] dup_rows_per_dst=[17804, 17502, 12240, 17126, 13974, 18954, 21454, 17184] combine_rows_per_src=[16204, 16082, 15342, 15766, 14790, 15818, 16232, 15672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125882 payload_rows_remote=109838 dup_rows=136262 dup_rows_remote=117618 dedup_ratio=0.519798 router_tokens_per_expert=[80, 1, 197, 3, 172, 0, 2864, 0, 1508, 451, 57, 3688, 4402, 5151, 0, 7411, 7238, 234, 512, 3852, 214, 6670, 219, 2743, 1, 96, 355, 4479, 3681, 12, 1072, 1684, 3827, 26, 548, 5850, 4317, 0, 3036, 2537, 2271, 6, 2398, 5321, 0, 410, 3392, 3721, 1315, 7012, 3884, 1, 1, 6, 36, 5940, 0, 2177, 262, 651, 2764, 4281, 5889, 146] payload_rows_per_dst=[16100, 16044, 15782, 15876, 16258, 15370, 14748, 15704] dup_rows_per_dst=[15658, 18644, 18220, 18584, 26348, 14554, 7618, 16636] combine_rows_per_src=[16100, 16044, 15782, 15876, 16258, 15370, 14748, 15704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=893583360 dispatch_meta_bytes=937152 combine_payload_bytes=893583360 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=894520512 ingress_bytes=1245904896 link_bytes=1245904896 token_bytes=1788103872 weight_movement_bytes=352321536 total_nvlink_bytes=2140425408 weight_to_token_ratio=0.197036 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.024621 gemm_active_groups=20 gemm_peak_group_rows=27264 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.976547351e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=904937472 dispatch_meta_bytes=910760 combine_payload_bytes=904937472 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=905848232 ingress_bytes=1257259008 link_bytes=1257259008 token_bytes=1810785704 weight_movement_bytes=352321536 total_nvlink_bytes=2163107240 weight_to_token_ratio=0.194568 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=18304 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.040125639e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125400 payload_rows_remote=110526 dup_rows=136744 dup_rows_remote=123440 dedup_ratio=0.521637 router_tokens_per_expert=[129, 10, 102, 13, 189, 3, 4164, 1, 1849, 423, 89, 3712, 4557, 5111, 2, 7156, 6192, 332, 276, 3703, 144, 6634, 256, 2721, 27, 256, 332, 4851, 4180, 7, 1208, 1386, 3709, 54, 553, 5606, 3721, 2, 3062, 2988, 1815, 46, 2854, 4534, 9, 560, 3518, 3220, 1617, 7494, 4378, 0, 18, 32, 25, 5779, 1, 2889, 372, 762, 2842, 3128, 5301, 168] payload_rows_per_dst=[16222, 15712, 15418, 15904, 14874, 15504, 16280, 15486] dup_rows_per_dst=[17534, 17936, 12714, 19802, 13304, 17608, 22406, 15440] combine_rows_per_src=[16222, 15712, 15418, 15904, 14874, 15504, 16280, 15486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=900235264 dispatch_meta_bytes=894840 combine_payload_bytes=900235264 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=901130104 ingress_bytes=1428717568 link_bytes=1428717568 token_bytes=1801365368 weight_movement_bytes=528482304 total_nvlink_bytes=2329847672 weight_to_token_ratio=0.293379 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=22016 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.000222005e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=872611840 dispatch_meta_bytes=903216 combine_payload_bytes=872611840 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=873515056 ingress_bytes=1048772608 link_bytes=1048772608 token_bytes=1746126896 weight_movement_bytes=176160768 total_nvlink_bytes=1922287664 weight_to_token_ratio=0.100887 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=22656 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.872688826e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=902266880 dispatch_meta_bytes=917008 combine_payload_bytes=902266880 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=903183888 ingress_bytes=1078427648 link_bytes=1078427648 token_bytes=1805450768 weight_movement_bytes=176160768 total_nvlink_bytes=1981611536 weight_to_token_ratio=0.097572 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=21376 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.038744671e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=899792896 dispatch_meta_bytes=909824 combine_payload_bytes=899792896 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=900702720 ingress_bytes=1428275200 link_bytes=1428275200 token_bytes=1800495616 weight_movement_bytes=528482304 total_nvlink_bytes=2328977920 weight_to_token_ratio=0.293520 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=23808 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.997744929e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123426 payload_rows_remote=107470 dup_rows=138718 dup_rows_remote=123670 dedup_ratio=0.529167 router_tokens_per_expert=[240, 16, 128, 2, 434, 0, 3627, 11, 3008, 387, 62, 3188, 3923, 3996, 0, 7048, 6840, 410, 773, 4127, 50, 6690, 331, 2408, 5, 340, 320, 4070, 3696, 19, 1562, 1219, 3391, 39, 1010, 6168, 3842, 14, 2926, 1982, 1611, 11, 3046, 6031, 3, 385, 3503, 3427, 1674, 7544, 4426, 17, 4, 14, 16, 5748, 3, 2106, 351, 813, 3167, 3178, 5422, 270] payload_rows_per_dst=[15956, 15774, 16178, 15898, 16314, 15504, 12284, 15518] dup_rows_per_dst=[15048, 19042, 23962, 18900, 25182, 17050, 4432, 15102] combine_rows_per_src=[15956, 15774, 16178, 15898, 16314, 15504, 12284, 15518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114900 dedup_ratio=0.500008 router_tokens_per_expert=[8184, 1, 0, 0, 173, 8178, 2, 8191, 3, 6190, 0, 21, 0, 0, 0, 8184, 8086, 1, 2, 0, 8190, 0, 0, 0, 3, 1, 8189, 1, 8192, 34, 0, 489, 0, 4, 8191, 0, 3, 8187, 8188, 192, 1170, 3, 1, 8187, 0, 1, 0, 7622, 0, 0, 8186, 1, 10, 0, 612, 0, 0, 1, 3, 2, 3, 8188, 1, 1] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16692, 12414, 16174, 17434, 16770, 17584, 17610, 16396] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=905428992 dispatch_meta_bytes=935864 combine_payload_bytes=905428992 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=906364856 ingress_bytes=1257750528 link_bytes=1257750528 token_bytes=1811793848 weight_movement_bytes=352321536 total_nvlink_bytes=2164115384 weight_to_token_ratio=0.194460 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.023674 gemm_active_groups=20 gemm_peak_group_rows=16768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.042877946e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114670 dedup_ratio=0.500053 router_tokens_per_expert=[8183, 1, 1, 3, 511, 8158, 0, 8186, 0, 5966, 2, 1, 0, 0, 1, 8188, 7933, 3, 0, 3, 8169, 0, 0, 7, 2, 1, 8189, 2, 8189, 5, 0, 693, 3, 0, 8187, 2, 3, 8186, 8185, 258, 373, 3, 3, 8185, 2, 1, 1, 7832, 1, 0, 8188, 4, 23, 3, 1031, 1, 11, 0, 4, 0, 4, 8176, 1, 4] payload_rows_per_dst=[16384, 16380, 16380, 16382, 16380, 16384, 16384, 16384] dup_rows_per_dst=[17330, 11936, 15850, 17780, 16898, 16416, 18488, 16388] combine_rows_per_src=[16384, 16380, 16380, 16382, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128204 payload_rows_remote=111820 dup_rows=133940 dup_rows_remote=117530 dedup_ratio=0.510941 router_tokens_per_expert=[8180, 0, 0, 0, 215, 8169, 1, 8192, 0, 6186, 2, 0, 1, 1, 0, 8192, 7666, 2, 0, 6, 8191, 0, 1, 0, 1, 0, 8190, 3, 8188, 0, 0, 563, 3, 0, 8188, 4, 2, 8187, 8186, 95, 915, 2, 2, 8188, 1, 1, 2, 7541, 0, 0, 8186, 3, 33, 3, 1373, 2, 1, 13, 0, 4, 1, 8184, 2, 0] payload_rows_per_dst=[13522, 16384, 16382, 16382, 16384, 16384, 16382, 16384] dup_rows_per_dst=[12774, 12380, 19280, 7110, 22678, 24118, 19190, 16410] combine_rows_per_src=[13522, 16384, 16382, 16382, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114656 dup_rows=131106 dup_rows_remote=113768 dedup_ratio=0.500130 router_tokens_per_expert=[8154, 2, 3, 0, 279, 8174, 1, 8186, 0, 7280, 4, 28, 0, 1, 1, 8183, 8164, 1, 3, 2, 8164, 0, 4, 16, 1, 6, 8183, 8, 8187, 11, 0, 464, 2, 3, 8179, 2, 13, 8183, 8185, 30, 153, 3, 0, 8187, 22, 2, 4, 7689, 1, 1, 8175, 0, 9, 1, 474, 1, 26, 13, 3, 10, 1, 8185, 1, 4] payload_rows_per_dst=[16368, 16384, 16380, 16382, 16382, 16384, 16374, 16384] dup_rows_per_dst=[16858, 14610, 16328, 17338, 16442, 15736, 17320, 16474] combine_rows_per_src=[16368, 16384, 16380, 16382, 16382, 16384, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114662 dup_rows=131100 dup_rows_remote=112560 dedup_ratio=0.500107 router_tokens_per_expert=[8179, 0, 1, 2, 629, 8182, 3, 8183, 6, 7056, 0, 27, 0, 0, 8, 8163, 7724, 3, 1, 25, 8183, 1, 11, 16, 1, 1, 8184, 0, 8185, 12, 1, 649, 4, 12, 8181, 4, 0, 8178, 8186, 253, 173, 0, 6, 8185, 0, 1, 0, 6985, 7, 10, 8168, 10, 12, 3, 1065, 0, 4, 1, 0, 2, 0, 8180, 0, 6] payload_rows_per_dst=[16376, 16382, 16384, 16376, 16382, 16382, 16382, 16380] dup_rows_per_dst=[17616, 14138, 15544, 17690, 16882, 14318, 18540, 16372] combine_rows_per_src=[16376, 16382, 16384, 16376, 16382, 16382, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=110486 dedup_ratio=0.500015 router_tokens_per_expert=[8186, 0, 0, 0, 947, 8186, 2, 8191, 0, 6954, 0, 3, 2, 1, 1, 8191, 8046, 0, 0, 6, 8188, 0, 0, 0, 1, 1, 8192, 1, 8191, 0, 1, 713, 1, 0, 8191, 0, 2, 8191, 8192, 96, 58, 0, 0, 8191, 2, 0, 0, 7162, 0, 0, 8189, 0, 24, 0, 577, 0, 1, 4, 0, 4, 0, 8183, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[18258, 20590, 16100, 17816, 9908, 14442, 17580, 16382] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=880394240 dispatch_meta_bytes=924560 combine_payload_bytes=880394240 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=881318800 ingress_bytes=1232715776 link_bytes=1232715776 token_bytes=1761713040 weight_movement_bytes=352321536 total_nvlink_bytes=2114034576 weight_to_token_ratio=0.199988 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.018939 gemm_active_groups=22 gemm_peak_group_rows=29440 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.902693785e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=918344 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940426056 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879933768 weight_movement_bytes=176160768 total_nvlink_bytes=2056094536 weight_to_token_ratio=0.093706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917376 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940326784 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879736192 weight_movement_bytes=176160768 total_nvlink_bytes=2055896960 weight_to_token_ratio=0.093716 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=916029440 dispatch_meta_bytes=917400 combine_payload_bytes=916029440 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=916946840 ingress_bytes=1092190208 link_bytes=1092190208 token_bytes=1832976280 weight_movement_bytes=176160768 total_nvlink_bytes=2009137048 weight_to_token_ratio=0.096106 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.115809262e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=913696 combine_payload_bytes=939261952 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940175648 ingress_bytes=1291583488 link_bytes=1291583488 token_bytes=1879437600 weight_movement_bytes=352321536 total_nvlink_bytes=2231759136 weight_to_token_ratio=0.187461 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232328399e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=900680 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940392008 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879883336 weight_movement_bytes=176160768 total_nvlink_bytes=2056044104 weight_to_token_ratio=0.093708 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=908888 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940219992 ingress_bytes=1115471872 link_bytes=1115471872 token_bytes=1879531096 weight_movement_bytes=176160768 total_nvlink_bytes=2055691864 weight_to_token_ratio=0.093726 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246176862e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114150 dedup_ratio=0.500046 router_tokens_per_expert=[8191, 3, 0, 0, 250, 8157, 0, 8192, 0, 7275, 0, 14, 0, 0, 4, 8090, 8011, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8186, 131, 8192, 20, 0, 1001, 0, 0, 8174, 0, 30, 8186, 8189, 269, 46, 0, 0, 8192, 25, 0, 0, 6935, 0, 0, 8190, 0, 17, 0, 690, 0, 22, 3, 3, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16376, 16384, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[16818, 14390, 16022, 18676, 16934, 14012, 17792, 16440] combine_rows_per_src=[16384, 16376, 16384, 16384, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129174 payload_rows_remote=112790 dup_rows=132970 dup_rows_remote=105616 dedup_ratio=0.507240 router_tokens_per_expert=[0, 0, 0, 8191, 0, 1, 8191, 0, 0, 1, 3827, 1, 3, 0, 0, 4, 0, 1, 5483, 1, 1, 0, 8191, 1, 117, 0, 1, 8188, 0, 0, 0, 2, 0, 8186, 0, 3, 8190, 0, 8191, 1, 1, 7, 8188, 7042, 1, 8188, 0, 12, 0, 2, 0, 0, 1, 8192, 0, 8191, 8103, 0, 0, 8187, 0, 1, 8179, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 14506, 16384, 16364] dup_rows_per_dst=[16382, 21818, 27354, 16614, 16394, 1824, 16388, 16196] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 14506, 16384, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=104670 dedup_ratio=0.500031 router_tokens_per_expert=[3, 0, 2, 8191, 7, 1, 8189, 0, 1, 3, 5502, 1, 0, 0, 4, 7, 0, 1, 5849, 2, 3, 1, 8187, 1, 8, 3, 1, 8185, 4, 1, 2, 4, 1, 8183, 2, 0, 8185, 5, 8186, 1, 2, 18, 8188, 5037, 2, 8150, 0, 0, 1, 0, 2, 1, 2, 8188, 0, 8189, 8185, 1, 4, 8185, 1, 2, 8184, 4] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16376, 16384, 16384, 16384] dup_rows_per_dst=[16402, 11028, 11704, 16400, 16374, 26410, 16382, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16376, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=915304 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940341096 ingress_bytes=1467908096 link_bytes=1467908096 token_bytes=1879766888 weight_movement_bytes=528482304 total_nvlink_bytes=2408249192 weight_to_token_ratio=0.281142 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219672603e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=113462 dedup_ratio=0.500008 router_tokens_per_expert=[8179, 0, 0, 0, 83, 8171, 0, 8188, 1, 7123, 2, 19, 0, 0, 0, 8174, 7927, 0, 0, 0, 8159, 0, 0, 0, 3, 2, 8192, 0, 8192, 19, 1, 885, 0, 0, 8187, 0, 5, 8188, 8189, 761, 293, 2, 0, 8188, 0, 0, 0, 6933, 1, 9, 8190, 0, 17, 0, 594, 0, 0, 0, 0, 3, 0, 8191, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[17612, 30630, 15788, 17242, 14568, 14450, 4570, 16214] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130936 payload_rows_remote=114552 dup_rows=131208 dup_rows_remote=114760 dedup_ratio=0.500519 router_tokens_per_expert=[4, 0, 0, 8188, 0, 0, 8188, 0, 0, 0, 5317, 5, 5, 4, 0, 0, 4, 0, 5415, 1, 2, 5, 8187, 0, 35, 4, 0, 8188, 4, 4, 0, 0, 1, 8187, 1, 8, 8188, 0, 8189, 4, 4, 11, 8184, 5669, 2, 8145, 2, 1, 0, 0, 0, 3, 4, 8186, 3, 8187, 8156, 1, 6, 8187, 0, 1, 8182, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16256, 16382, 16378] dup_rows_per_dst=[16376, 12356, 14134, 16448, 16386, 22798, 16384, 16326] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16256, 16382, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=113860 dedup_ratio=0.500015 router_tokens_per_expert=[3, 0, 2, 8184, 2, 0, 8188, 2, 0, 2, 4517, 0, 0, 2, 2, 0, 0, 1, 6116, 2, 1, 1, 8188, 0, 1, 0, 0, 8190, 1, 2, 3, 2, 2, 8181, 9, 1, 8187, 4, 8187, 1, 3, 21, 8190, 5797, 2, 8118, 1, 0, 0, 0, 4, 1, 2, 8189, 5, 8189, 8183, 3, 0, 8188, 0, 4, 8188, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16356, 9036, 12234, 15856, 16386, 27600, 16392, 17216] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127380 payload_rows_remote=110996 dup_rows=134764 dup_rows_remote=105418 dedup_ratio=0.514084 router_tokens_per_expert=[0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 6479, 0, 1, 0, 0, 1, 0, 0, 3582, 0, 0, 0, 8192, 0, 3, 2, 1, 8192, 0, 0, 0, 0, 0, 8192, 0, 1, 8192, 2, 8192, 0, 0, 28, 8191, 6331, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 8188, 0, 0, 8190, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 12694, 16384, 16382] dup_rows_per_dst=[16382, 29346, 23482, 16394, 16378, 24, 16382, 16376] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 12694, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=923975680 dispatch_meta_bytes=873624 combine_payload_bytes=923975680 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=924849304 ingress_bytes=1100136448 link_bytes=1100136448 token_bytes=1848824984 weight_movement_bytes=176160768 total_nvlink_bytes=2024985752 weight_to_token_ratio=0.095283 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.160304889e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114700 dedup_ratio=0.500038 router_tokens_per_expert=[2, 8, 3, 8191, 0, 0, 8191, 0, 4, 1, 4794, 0, 4, 0, 0, 4, 0, 2, 5392, 3, 1, 0, 8191, 0, 17, 8, 0, 8187, 3, 3, 0, 0, 0, 8192, 0, 1, 8191, 1, 8188, 0, 0, 31, 8185, 6248, 0, 8093, 2, 26, 1, 0, 6, 0, 0, 8191, 0, 8185, 8162, 0, 0, 8183, 0, 3, 8173, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] dup_rows_per_dst=[16406, 9612, 10794, 16416, 16380, 28786, 16382, 16306] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=877400 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940335960 ingress_bytes=1556021248 link_bytes=1556021248 token_bytes=1879794520 weight_movement_bytes=616562688 total_nvlink_bytes=2496357208 weight_to_token_ratio=0.327995 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=25088 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713069474e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=912592 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940420304 ingress_bytes=1644150784 link_bytes=1644150784 token_bytes=1879928016 weight_movement_bytes=704643072 total_nvlink_bytes=2584571088 weight_to_token_ratio=0.374824 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206558088e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=938409984 dispatch_meta_bytes=917248 combine_payload_bytes=938409984 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939327232 ingress_bytes=1114570752 link_bytes=1114570752 token_bytes=1877737216 weight_movement_bytes=176160768 total_nvlink_bytes=2053897984 weight_to_token_ratio=0.093815 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.241130966e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=914176 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940405504 ingress_bytes=1379893248 link_bytes=1379893248 token_bytes=1879896832 weight_movement_bytes=440401920 total_nvlink_bytes=2320298752 weight_to_token_ratio=0.234269 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726826193e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=909279232 dispatch_meta_bytes=865656 combine_payload_bytes=909279232 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=910144888 ingress_bytes=1261600768 link_bytes=1261600768 token_bytes=1819424120 weight_movement_bytes=352321536 total_nvlink_bytes=2171745656 weight_to_token_ratio=0.193645 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.064437683e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917512 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940359688 ingress_bytes=1203683328 link_bytes=1203683328 token_bytes=1879801864 weight_movement_bytes=264241152 total_nvlink_bytes=2144043016 weight_to_token_ratio=0.140569 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740124195e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114674 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0, 0, 12, 4864, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 8192, 0, 84, 4, 2, 8186, 0, 0, 0, 0, 0, 8192, 0, 7, 8192, 0, 8192, 0, 0, 348, 8176, 7468, 0, 8185, 0, 13, 0, 0, 0, 0, 0, 8192, 0, 8192, 8048, 0, 3, 8188, 2, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 9752, 7508, 16552, 16398, 31996, 16384, 16098] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114698 dedup_ratio=0.500008 router_tokens_per_expert=[1, 1, 0, 0, 4, 1, 1, 0, 0, 1, 1, 1, 0, 2, 0, 0, 0, 0, 8191, 1, 1, 1, 8185, 1, 0, 0, 0, 8190, 0, 1, 0, 1, 2, 1, 8157, 0, 8191, 8191, 0, 0, 0, 8191, 0, 135, 0, 0, 8188, 8191, 8191, 8191, 1, 0, 8191, 8191, 8191, 1, 3, 8191, 77, 8011, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16396, 16390, 16376, 16382, 16318, 16646, 16384, 16182] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114700 dedup_ratio=0.500038 router_tokens_per_expert=[2, 2, 0, 2, 4, 2, 2, 0, 0, 3, 0, 1, 2, 9, 1, 2, 0, 0, 8189, 5, 2, 1, 8186, 1, 0, 2, 0, 8178, 1, 2, 1, 2, 4, 2, 8184, 2, 8187, 8190, 2, 2, 0, 8187, 2, 4, 0, 0, 8187, 8190, 8186, 8179, 2, 0, 8187, 8188, 8189, 0, 4, 8188, 18, 8184, 0, 1, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16378, 16384] dup_rows_per_dst=[16402, 16404, 16384, 16364, 16382, 16382, 16356, 16408] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917448 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441544 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965640 weight_movement_bytes=528482304 total_nvlink_bytes=2408447944 weight_to_token_ratio=0.281113 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128114 payload_rows_remote=111730 dup_rows=134030 dup_rows_remote=117642 dedup_ratio=0.511284 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 5869, 0, 0, 0, 0, 1, 0, 0, 3741, 0, 0, 0, 8192, 0, 8, 6, 0, 8186, 0, 0, 0, 0, 0, 8187, 0, 6, 8192, 0, 8192, 0, 0, 342, 8191, 6449, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8179, 0, 0, 8184, 0, 0, 8189, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 13438, 16384, 16372] dup_rows_per_dst=[16388, 28124, 23862, 16392, 16384, 276, 16380, 16224] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 13438, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114656 dedup_ratio=0.500038 router_tokens_per_expert=[0, 4, 0, 0, 9, 1, 5, 7, 1, 3, 4, 1, 4, 2, 0, 0, 0, 6, 8181, 3, 5, 5, 8155, 0, 1, 0, 1, 8183, 0, 5, 2, 27, 4, 2, 8158, 8, 8187, 8183, 0, 4, 0, 8183, 1, 71, 0, 0, 8180, 8185, 8186, 8188, 5, 0, 8168, 8187, 8187, 5, 9, 8188, 55, 8108, 0, 0, 0, 5] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16376, 16384, 16384] dup_rows_per_dst=[16410, 16390, 16328, 16426, 16342, 16494, 16346, 16346] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917536 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425248 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932960 weight_movement_bytes=176160768 total_nvlink_bytes=2056093728 weight_to_token_ratio=0.093706 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 1, 0, 0, 8184, 0, 1, 0, 0, 8190, 0, 0, 0, 54, 1, 0, 8186, 0, 8192, 8192, 0, 0, 0, 8191, 0, 35, 0, 0, 8190, 8192, 8192, 8192, 0, 0, 8192, 8192, 8192, 0, 1, 8191, 30, 8088, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 16384, 16368, 16490, 16374, 16450, 16384, 16236] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114684 dup_rows=131080 dup_rows_remote=113184 dedup_ratio=0.500031 router_tokens_per_expert=[4, 5, 0, 3, 0, 3, 3, 3, 3, 2, 3, 2, 2, 0, 0, 0, 0, 1, 8188, 0, 5, 3, 8183, 2, 0, 0, 1, 8188, 1, 4, 2, 43, 0, 3, 8189, 0, 8188, 8185, 1, 3, 0, 8188, 3, 0, 0, 0, 8188, 8187, 8187, 8179, 4, 0, 8176, 8188, 8187, 3, 0, 8188, 0, 8169, 0, 0, 0, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16382, 16380] dup_rows_per_dst=[16296, 16200, 16380, 15704, 14820, 16598, 17186, 17896] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917512 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940359688 ingress_bytes=1467924480 link_bytes=1467924480 token_bytes=1879801864 weight_movement_bytes=528482304 total_nvlink_bytes=2408284168 weight_to_token_ratio=0.281137 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219764346e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114702 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 1, 0, 12, 0, 0, 0, 1, 0, 0, 0, 1, 4, 0, 6, 2, 0, 8191, 1, 0, 0, 8173, 0, 0, 7, 9, 8180, 0, 0, 0, 162, 8, 0, 8172, 4, 8188, 8191, 0, 2, 0, 8182, 0, 23, 0, 0, 8187, 8192, 8189, 8188, 0, 0, 8191, 8184, 8191, 3, 10, 8185, 27, 8003, 1, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16400, 16388, 16350, 16710, 16364, 16420, 16372, 16070] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=915292160 dispatch_meta_bytes=917488 combine_payload_bytes=915292160 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=916209648 ingress_bytes=1179533312 link_bytes=1179533312 token_bytes=1831501808 weight_movement_bytes=264241152 total_nvlink_bytes=2095742960 weight_to_token_ratio=0.144276 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.604894186e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917336 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940359512 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879801688 weight_movement_bytes=176160768 total_nvlink_bytes=2055962456 weight_to_token_ratio=0.093712 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=911472 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940402800 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879894128 weight_movement_bytes=176160768 total_nvlink_bytes=2056054896 weight_to_token_ratio=0.093708 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.023674 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 2, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114884 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 8192, 1, 0, 0, 8117, 0, 0, 0, 0, 8191, 0, 0, 0, 122, 9, 0, 8085, 0, 8192, 8192, 0, 0, 0, 8192, 0, 398, 0, 0, 8117, 8192, 8192, 8192, 0, 0, 8192, 8192, 8192, 0, 56, 8192, 149, 7643, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16524, 16236, 16626, 16188, 17030, 16384, 15696] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917552 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=940425264 ingress_bytes=1820311552 link_bytes=1820311552 token_bytes=1879932976 weight_movement_bytes=880803840 total_nvlink_bytes=2760736816 weight_to_token_ratio=0.468529 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019298486e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114688 dup_rows=131092 dup_rows_remote=118490 dedup_ratio=0.500076 router_tokens_per_expert=[8191, 8191, 7744, 0, 0, 0, 1, 1, 8189, 0, 8190, 1, 1, 1, 0, 2, 0, 0, 0, 8170, 6307, 0, 6, 0, 2, 8191, 1, 0, 8191, 0, 1, 8177, 8191, 8191, 1, 0, 8191, 4, 0, 1, 129, 8191, 0, 1, 2231, 1, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 1, 0, 1, 8188, 0, 0, 1, 0, 1] payload_rows_per_dst=[16384, 16384, 16364, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[30650, 16384, 12602, 16360, 16392, 5944, 16380, 16380] combine_rows_per_src=[16384, 16384, 16364, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130722 payload_rows_remote=114338 dup_rows=131422 dup_rows_remote=107500 dedup_ratio=0.501335 router_tokens_per_expert=[8153, 8190, 7888, 1, 0, 1, 2, 2, 8185, 0, 8190, 1, 2, 2, 0, 1, 1, 0, 1, 8183, 4756, 0, 3, 1, 0, 8186, 0, 0, 8188, 1, 0, 8189, 8189, 8189, 4, 3, 8190, 4, 1, 1, 582, 8186, 0, 2, 3189, 2, 2, 0, 2, 2, 2, 1, 8190, 2, 6, 1, 0, 2, 8183, 3, 0, 1, 4, 2] payload_rows_per_dst=[16054, 16384, 16370, 16380, 16382, 16384, 16384, 16384] dup_rows_per_dst=[11302, 16378, 14258, 16368, 16400, 23922, 16408, 16386] combine_rows_per_src=[16054, 16384, 16370, 16380, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918288 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940442384 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879966480 weight_movement_bytes=528482304 total_nvlink_bytes=2408448784 weight_to_token_ratio=0.281113 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114728 dedup_ratio=0.500061 router_tokens_per_expert=[8134, 8186, 7930, 1, 0, 0, 5, 3, 8188, 3, 8185, 5, 5, 6, 0, 4, 0, 4, 4, 8162, 6021, 0, 8, 0, 2, 8183, 0, 0, 8179, 10, 0, 8173, 8186, 8186, 6, 0, 8186, 13, 0, 0, 42, 8187, 0, 5, 2450, 4, 5, 0, 0, 0, 5, 0, 8184, 0, 9, 6, 0, 4, 8182, 0, 0, 5, 0, 6] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16374, 16384, 16380, 16384] dup_rows_per_dst=[16736, 16408, 12014, 16360, 16408, 20402, 16378, 16382] combine_rows_per_src=[16382, 16384, 16384, 16384, 16374, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=932712 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940456808 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879980904 weight_movement_bytes=176160768 total_nvlink_bytes=2056141672 weight_to_token_ratio=0.093703 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130768 payload_rows_remote=114384 dup_rows=131376 dup_rows_remote=114982 dedup_ratio=0.501160 router_tokens_per_expert=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 8191, 3, 0, 0, 8170, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 8180, 0, 8192, 8192, 0, 0, 0, 8191, 0, 103, 0, 0, 8187, 8192, 8192, 8192, 0, 0, 8192, 8192, 8192, 0, 9, 8192, 75, 8037, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16080, 16384] dup_rows_per_dst=[16394, 16392, 16652, 16374, 18284, 15066, 15972, 16242] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16080, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114690 dedup_ratio=0.500008 router_tokens_per_expert=[8158, 8192, 7370, 6, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 6904, 0, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 1, 0, 8192, 0, 0, 0, 65, 8191, 0, 0, 2079, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[31068, 16384, 13796, 16382, 16386, 4286, 16388, 16384] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129922 payload_rows_remote=113538 dup_rows=132222 dup_rows_remote=115786 dedup_ratio=0.504387 router_tokens_per_expert=[8189, 8187, 7640, 15, 0, 1, 4, 4, 8188, 3, 8185, 4, 5, 4, 0, 2, 0, 0, 3, 8188, 4868, 1, 7, 1, 1, 8189, 0, 0, 8183, 5, 0, 8176, 8187, 8187, 0, 0, 8187, 10, 0, 3, 505, 8188, 0, 5, 3344, 4, 1, 0, 3, 1, 3, 0, 8185, 0, 0, 5, 0, 2, 8188, 0, 2, 4, 0, 5] payload_rows_per_dst=[15242, 16384, 16384, 16382, 16378, 16384, 16384, 16384] dup_rows_per_dst=[30, 16398, 26130, 16354, 16408, 24084, 16382, 16436] combine_rows_per_src=[15242, 16384, 16384, 16382, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=936656896 dispatch_meta_bytes=887352 combine_payload_bytes=936656896 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=937544248 ingress_bytes=1112817664 link_bytes=1112817664 token_bytes=1874201144 weight_movement_bytes=176160768 total_nvlink_bytes=2050361912 weight_to_token_ratio=0.093992 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.231314405e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917600 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940310624 ingress_bytes=1291714560 link_bytes=1291714560 token_bytes=1879703648 weight_movement_bytes=352321536 total_nvlink_bytes=2232025184 weight_to_token_ratio=0.187435 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233062348e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129700 payload_rows_remote=113316 dup_rows=132444 dup_rows_remote=116034 dedup_ratio=0.505234 router_tokens_per_expert=[8118, 8192, 7412, 31, 0, 0, 0, 0, 8183, 0, 8191, 0, 0, 0, 0, 0, 4, 0, 0, 8165, 6399, 0, 0, 0, 0, 8180, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 11, 8, 8190, 0, 11, 0, 325, 8180, 0, 0, 2315, 0, 0, 0, 0, 12, 0, 1, 8192, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 4, 0] payload_rows_per_dst=[15058, 16382, 16340, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[328, 16366, 28532, 16360, 16440, 21640, 16410, 16368] combine_rows_per_src=[15058, 16382, 16340, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=937033728 dispatch_meta_bytes=917464 combine_payload_bytes=937033728 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937951192 ingress_bytes=1289355264 link_bytes=1289355264 token_bytes=1874984920 weight_movement_bytes=352321536 total_nvlink_bytes=2227306456 weight_to_token_ratio=0.187906 gemm_rows_defined=132992 gemm_rows_charged=135168 gemm_tail_pad_rows=2176 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.016098 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.219851275e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940425216 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932928 weight_movement_bytes=264241152 total_nvlink_bytes=2144174080 weight_to_token_ratio=0.140559 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=930103296 dispatch_meta_bytes=917296 combine_payload_bytes=930103296 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=931020592 ingress_bytes=1282424832 link_bytes=1282424832 token_bytes=1861123888 weight_movement_bytes=352321536 total_nvlink_bytes=2213445424 weight_to_token_ratio=0.189306 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.181043749e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114678 dedup_ratio=0.500061 router_tokens_per_expert=[7996, 8192, 6712, 67, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 6652, 0, 0, 0, 0, 8169, 0, 0, 8192, 0, 0, 8192, 8192, 8192, 13, 0, 8192, 0, 0, 0, 343, 8192, 0, 0, 2940, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16384, 16376, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[13174, 16384, 13088, 16338, 16410, 22950, 16386, 16358] combine_rows_per_src=[16376, 16384, 16376, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114684 dup_rows=131078 dup_rows_remote=118678 dedup_ratio=0.500023 router_tokens_per_expert=[2, 0, 8191, 0, 0, 1, 0, 8185, 8187, 0, 0, 1, 8191, 0, 0, 8, 6072, 1, 1, 8182, 0, 1, 8191, 1, 8191, 8189, 1, 0, 2088, 0, 0, 1, 8191, 8191, 0, 1, 0, 3, 4, 0, 8159, 1, 0, 8191, 0, 1, 19, 8191, 1, 0, 0, 0, 0, 0, 8190, 1, 8191, 1, 0, 0, 9, 41, 1, 0] payload_rows_per_dst=[16384, 16384, 16382, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16374, 16390, 12400, 20294, 16396, 16358, 16382, 16484] combine_rows_per_src=[16384, 16384, 16382, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=928284672 dispatch_meta_bytes=917400 combine_payload_bytes=928284672 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=929202072 ingress_bytes=1104445440 link_bytes=1104445440 token_bytes=1857486744 weight_movement_bytes=176160768 total_nvlink_bytes=2033647512 weight_to_token_ratio=0.094838 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.184433446e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114680 dup_rows=131084 dup_rows_remote=114678 dedup_ratio=0.500046 router_tokens_per_expert=[2, 1, 8190, 0, 0, 1, 0, 8115, 8170, 0, 0, 2, 8188, 2, 0, 57, 7356, 2, 1, 8183, 1, 3, 8188, 1, 8190, 8177, 6, 1, 836, 1, 0, 2, 8190, 8181, 0, 2, 0, 5, 0, 5, 8184, 1, 0, 8189, 0, 3, 18, 8187, 2, 1, 1, 5, 0, 2, 8189, 2, 8190, 2, 3, 1, 21, 5, 1, 6] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[16234, 16454, 14718, 18042, 16382, 16406, 16398, 16450] combine_rows_per_src=[16384, 16384, 16376, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917400 combine_payload_bytes=939393024 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=940310424 ingress_bytes=1644036096 link_bytes=1644036096 token_bytes=1879703448 weight_movement_bytes=704643072 total_nvlink_bytes=2584346520 weight_to_token_ratio=0.374869 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205915883e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114650 dup_rows=131110 dup_rows_remote=109148 dedup_ratio=0.500145 router_tokens_per_expert=[5, 0, 8185, 0, 0, 4, 3, 8092, 8174, 0, 0, 5, 8180, 6, 0, 11, 5407, 6, 2, 8158, 0, 6, 8183, 6, 8186, 8182, 3, 0, 2794, 1, 0, 7, 8186, 8181, 0, 4, 0, 9, 6, 2, 8174, 4, 0, 8186, 0, 10, 57, 8186, 5, 3, 0, 15, 0, 4, 8185, 6, 8185, 6, 5, 0, 15, 27, 1, 4] payload_rows_per_dst=[16384, 16378, 16352, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16194, 16374, 10818, 21962, 16392, 16478, 16424, 16468] combine_rows_per_src=[16384, 16378, 16352, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=933448 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940424776 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879916104 weight_movement_bytes=352321536 total_nvlink_bytes=2232237640 weight_to_token_ratio=0.187413 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114688 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8190, 8185, 0, 0, 0, 8192, 0, 0, 11, 6335, 0, 0, 8189, 0, 0, 8192, 0, 8192, 8188, 0, 0, 1854, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 8192, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 7, 5, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16392, 25348, 7408, 16386, 16374, 16384, 16408] combine_rows_per_src=[16384, 16384, 16384, 16376, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=100488 dedup_ratio=0.500031 router_tokens_per_expert=[8120, 8192, 7164, 5, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 5397, 0, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 8188, 8192, 8192, 0, 0, 8192, 0, 0, 0, 273, 8192, 4, 0, 3645, 0, 0, 0, 0, 3, 0, 0, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[30592, 16372, 10746, 16556, 16248, 7844, 16344, 16378] combine_rows_per_src=[16384, 16384, 16378, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114706 dedup_ratio=0.500031 router_tokens_per_expert=[4, 0, 8187, 0, 0, 4, 0, 8138, 8187, 0, 0, 4, 8182, 2, 0, 1, 7321, 5, 3, 8182, 0, 5, 8184, 3, 8187, 8189, 0, 3, 902, 1, 0, 3, 8187, 8188, 2, 4, 0, 0, 2, 0, 8189, 4, 0, 8187, 0, 8, 6, 8187, 6, 3, 0, 14, 0, 2, 8186, 5, 8187, 5, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16382, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16284, 16130, 14658, 18184, 16384, 16644, 16422, 16374] combine_rows_per_src=[16382, 16382, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917432 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940375992 ingress_bytes=1467940864 link_bytes=1467940864 token_bytes=1879834552 weight_movement_bytes=528482304 total_nvlink_bytes=2408316856 weight_to_token_ratio=0.281132 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219856090e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=895192 combine_payload_bytes=939212800 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940107992 ingress_bytes=1379614720 link_bytes=1379614720 token_bytes=1879320792 weight_movement_bytes=440401920 total_nvlink_bytes=2319722712 weight_to_token_ratio=0.234341 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725266552e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917472 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940376032 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879834592 weight_movement_bytes=352321536 total_nvlink_bytes=2232156128 weight_to_token_ratio=0.187422 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=860672 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940319232 ingress_bytes=1556021248 link_bytes=1556021248 token_bytes=1879777792 weight_movement_bytes=616562688 total_nvlink_bytes=2496340480 weight_to_token_ratio=0.327998 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.017992 gemm_active_groups=18 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713069474e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114698 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8144, 8175, 0, 2, 0, 8192, 0, 0, 20, 5284, 0, 0, 8184, 6, 0, 8192, 0, 8192, 8177, 11, 0, 2886, 0, 0, 0, 8192, 8185, 6, 0, 0, 2, 0, 22, 8171, 0, 0, 8192, 0, 0, 3, 8188, 0, 1, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 1, 41, 25, 0, 2] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16376, 16384, 16384] dup_rows_per_dst=[16288, 16394, 10568, 22148, 16430, 16348, 16386, 16522] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917544 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940376104 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834664 weight_movement_bytes=176160768 total_nvlink_bytes=2055995432 weight_to_token_ratio=0.093711 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130932 payload_rows_remote=114548 dup_rows=131212 dup_rows_remote=114832 dedup_ratio=0.500534 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8139, 8184, 0, 0, 0, 8192, 0, 0, 46, 6408, 0, 0, 8120, 0, 0, 8192, 0, 8192, 8190, 4, 0, 1630, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 8192, 0, 0, 70, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 2, 131, 81, 0, 0] payload_rows_per_dst=[16384, 16384, 16244, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16278, 16460, 12812, 19648, 16380, 16438, 16384, 16812] combine_rows_per_src=[16384, 16384, 16244, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114686 dup_rows=131080 dup_rows_remote=114830 dedup_ratio=0.500031 router_tokens_per_expert=[1, 0, 1, 1, 0, 1, 8190, 8187, 1, 8191, 3, 2, 1, 0, 0, 1, 1, 8188, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 1101, 0, 0, 1, 1, 1, 0, 3, 1, 1, 0, 8191, 0, 8188, 8190, 8191, 8191, 0, 1, 5886, 1, 0, 8190, 0, 1, 0, 15, 0, 8191, 8190, 1, 1, 8191, 0, 3, 8190, 1257] payload_rows_per_dst=[16382, 16384, 16378, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16394, 16250, 18580, 16390, 13686, 16410, 16990] combine_rows_per_src=[16382, 16384, 16378, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=121894 dedup_ratio=0.500015 router_tokens_per_expert=[3, 0, 3, 2, 0, 3, 8186, 8181, 1, 8186, 5, 1, 3, 3, 2, 3, 0, 8186, 2, 9, 0, 0, 3, 8118, 1, 0, 0, 1691, 1, 14, 2, 0, 1, 1, 7, 2, 2, 0, 8188, 0, 8184, 8187, 8162, 8188, 0, 0, 3845, 2, 1, 8189, 1, 2, 1, 3, 5, 8188, 8189, 2, 2, 8189, 0, 0, 8189, 2733] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16372, 16398, 16254, 18230, 16396, 9182, 16396, 21848] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917496 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940343288 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879769080 weight_movement_bytes=176160768 total_nvlink_bytes=2055929848 weight_to_token_ratio=0.093714 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938377216 dispatch_meta_bytes=917520 combine_payload_bytes=938377216 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=939294736 ingress_bytes=1202618368 link_bytes=1202618368 token_bytes=1877671952 weight_movement_bytes=264241152 total_nvlink_bytes=2141913104 weight_to_token_ratio=0.140728 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.734160863e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130944 payload_rows_remote=114560 dup_rows=131200 dup_rows_remote=111514 dedup_ratio=0.500488 router_tokens_per_expert=[7, 0, 9, 6, 0, 7, 8179, 8180, 4, 8179, 8, 1, 6, 6, 0, 6, 0, 8183, 2, 2, 0, 7, 0, 8004, 0, 0, 0, 1657, 8, 0, 5, 0, 0, 0, 5, 8, 6, 0, 8186, 0, 8180, 8185, 8092, 8185, 0, 7, 3678, 6, 0, 8184, 0, 5, 0, 19, 21, 8186, 8185, 6, 6, 8184, 0, 0, 8184, 3088] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16262, 16382, 16380] dup_rows_per_dst=[16392, 16406, 16012, 19686, 16396, 7304, 16448, 22556] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16262, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 4, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=918064 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425776 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879933488 weight_movement_bytes=176160768 total_nvlink_bytes=2056094256 weight_to_token_ratio=0.093706 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114690 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8191, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 8147, 0, 0, 0, 2507, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8191, 8192, 8191, 8192, 0, 0, 3559, 0, 0, 8192, 0, 0, 0, 8, 0, 8192, 8192, 0, 0, 8192, 0, 0, 8192, 2165] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 16382, 16296, 21396, 16386, 21676, 16400, 6154] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130954 payload_rows_remote=114570 dup_rows=131190 dup_rows_remote=114148 dedup_ratio=0.500450 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 8183, 8179, 0, 0, 0, 8192, 0, 0, 25, 6367, 0, 0, 8154, 0, 0, 8192, 0, 8192, 8185, 0, 0, 1799, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 9, 8170, 0, 0, 8192, 0, 0, 48, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 4, 6, 24, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16266, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17042, 16486, 29940, 3516, 16322, 16038, 16292, 15554] combine_rows_per_src=[16384, 16384, 16384, 16266, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=946312 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940437640 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=1879928968 weight_movement_bytes=704643072 total_nvlink_bytes=2584572040 weight_to_token_ratio=0.374824 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114676 dup_rows=131088 dup_rows_remote=107242 dedup_ratio=0.500061 router_tokens_per_expert=[7, 0, 9, 5, 1, 5, 8176, 8187, 1, 8187, 0, 3, 2, 5, 0, 5, 0, 8182, 3, 0, 0, 1, 0, 8129, 0, 1, 0, 2160, 5, 3, 5, 0, 3, 0, 0, 6, 5, 0, 8187, 0, 8187, 8185, 8113, 8187, 1, 3, 2593, 5, 0, 8185, 8, 4, 0, 0, 5, 8187, 8186, 5, 5, 8185, 0, 0, 8186, 3559] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16384, 16384, 16382, 16380] dup_rows_per_dst=[16396, 16324, 16256, 4338, 16336, 21488, 16104, 23846] combine_rows_per_src=[16384, 16384, 16374, 16384, 16384, 16384, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938475520 dispatch_meta_bytes=904296 combine_payload_bytes=938475520 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939379816 ingress_bytes=1290797056 link_bytes=1290797056 token_bytes=1877855336 weight_movement_bytes=352321536 total_nvlink_bytes=2230176872 weight_to_token_ratio=0.187619 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.227924708e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 2, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917512 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441608 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965704 weight_movement_bytes=176160768 total_nvlink_bytes=2056126472 weight_to_token_ratio=0.093704 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938557440 dispatch_meta_bytes=914872 combine_payload_bytes=938557440 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939472312 ingress_bytes=1114718208 link_bytes=1114718208 token_bytes=1878029752 weight_movement_bytes=176160768 total_nvlink_bytes=2054190520 weight_to_token_ratio=0.093801 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.241956659e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=887672 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940313464 ingress_bytes=1644068864 link_bytes=1644068864 token_bytes=1879739256 weight_movement_bytes=704643072 total_nvlink_bytes=2584382328 weight_to_token_ratio=0.374862 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206099370e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114660 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8184, 0, 8185, 2, 0, 0, 0, 3, 0, 0, 8180, 0, 19, 0, 0, 4, 8155, 0, 0, 0, 1168, 0, 5, 0, 0, 0, 0, 3, 0, 0, 4, 8187, 0, 8187, 8189, 8192, 8192, 0, 0, 4510, 0, 5, 8192, 0, 0, 2, 14, 0, 8186, 8192, 0, 0, 8192, 0, 6, 8192, 2530] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16368, 16376, 16332, 5948, 16388, 21792, 16414, 21456] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114678 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8168, 0, 8191, 12, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 2742, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 8191, 8192, 8192, 8192, 0, 0, 4656, 0, 0, 8192, 0, 0, 0, 85, 0, 8192, 8192, 0, 0, 8192, 0, 0, 8192, 808] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16336, 16406, 16204, 21866, 16394, 9312, 16554, 18000] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=110832 dedup_ratio=0.500000 router_tokens_per_expert=[1, 0, 0, 8191, 7383, 1, 3, 1, 1, 8191, 1, 1, 0, 30, 1, 0, 1964, 8191, 8187, 1, 0, 8190, 0, 8160, 0, 8191, 0, 8191, 0, 0, 0, 1, 1, 8191, 0, 0, 1, 0, 0, 7035, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 8191, 1, 0, 1, 8191, 0, 8191, 1, 0, 1, 8191, 8190] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14776, 16448, 20240, 16382, 14072, 16386, 16386, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 0, 0, 0, 2, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[1, 0, 0, 8189, 7297, 2, 3, 8, 2, 8184, 3, 2, 0, 58, 3, 0, 1669, 8189, 8183, 3, 1, 8188, 2, 8179, 0, 8189, 0, 8188, 1, 0, 10, 2, 3, 8187, 0, 1, 3, 0, 1, 7357, 0, 0, 0, 1, 2, 2, 0, 2, 0, 0, 3, 2, 8189, 2, 0, 1, 8189, 1, 8189, 2, 0, 2, 8188, 8189] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14616, 16498, 19690, 16396, 14720, 16384, 16388, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114652 dedup_ratio=0.500000 router_tokens_per_expert=[2, 0, 0, 8184, 7260, 6, 6, 3, 7, 8181, 7, 6, 0, 36, 7, 0, 3819, 8185, 8181, 7, 0, 8184, 1, 8159, 0, 8184, 4, 8184, 0, 2, 23, 5, 13, 8186, 0, 0, 8, 0, 4, 5252, 0, 0, 0, 0, 6, 6, 0, 2, 0, 3, 7, 0, 8183, 6, 0, 4, 8185, 0, 8185, 7, 1, 8, 8182, 8181] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14538, 16474, 23950, 16420, 10542, 16382, 16392, 16374] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917384 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940425096 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932808 weight_movement_bytes=352321536 total_nvlink_bytes=2232254344 weight_to_token_ratio=0.187412 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917464 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441560 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965656 weight_movement_bytes=176160768 total_nvlink_bytes=2056126424 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=902080 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940426176 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879950272 weight_movement_bytes=880803840 total_nvlink_bytes=2760754112 weight_to_token_ratio=0.468525 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.017992 gemm_active_groups=18 gemm_peak_group_rows=29312 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114606 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 7099, 0, 1, 1, 0, 8190, 0, 0, 0, 43, 0, 0, 2470, 8192, 8191, 0, 0, 8192, 0, 8191, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 6775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14202, 16466, 21320, 16384, 13548, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130638 payload_rows_remote=114254 dup_rows=131506 dup_rows_remote=115058 dedup_ratio=0.501656 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 8191, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 8162, 0, 0, 0, 2326, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8188, 8192, 8191, 8192, 0, 0, 5186, 0, 0, 8192, 0, 0, 0, 16, 0, 8192, 8192, 0, 0, 8192, 0, 0, 8192, 695] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 15950] dup_rows_per_dst=[16448, 16388, 17486, 21028, 16386, 26754, 16350, 666] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 15950] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917360 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940441456 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965552 weight_movement_bytes=176160768 total_nvlink_bytes=2056126320 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113778 dedup_ratio=0.500000 router_tokens_per_expert=[4, 0, 0, 8185, 7701, 5, 0, 0, 5, 8187, 5, 5, 0, 9, 5, 0, 1952, 8184, 8187, 6, 3, 8186, 1, 8187, 0, 8186, 0, 8185, 1, 3, 30, 4, 4, 8187, 0, 0, 2, 0, 5, 6677, 0, 0, 1, 0, 5, 5, 0, 5, 1, 3, 6, 0, 8186, 4, 0, 4, 8187, 0, 8186, 5, 0, 5, 8184, 8184] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15406, 15622, 3914, 16338, 29740, 16388, 16370, 17294] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917176 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441272 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965368 weight_movement_bytes=176160768 total_nvlink_bytes=2056126136 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=935968768 dispatch_meta_bytes=917248 combine_payload_bytes=935968768 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=936886016 ingress_bytes=1112129536 link_bytes=1112129536 token_bytes=1872854784 weight_movement_bytes=176160768 total_nvlink_bytes=2049015552 weight_to_token_ratio=0.094060 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.227461176e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=913864 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940437960 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879962056 weight_movement_bytes=704643072 total_nvlink_bytes=2584605128 weight_to_token_ratio=0.374818 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=119400 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 7168, 0, 0, 0, 0, 8192, 0, 0, 0, 107, 0, 0, 3367, 8192, 8189, 0, 0, 8192, 0, 8101, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14336, 16598, 22930, 16384, 11672, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 7995, 0, 1, 9, 0, 8177, 0, 0, 0, 33, 0, 0, 2789, 8192, 8183, 0, 0, 8192, 0, 8156, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 5618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 6, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16010, 16420, 8474, 16384, 24620, 16384, 16384, 16396] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130862 payload_rows_remote=114478 dup_rows=131282 dup_rows_remote=114892 dedup_ratio=0.500801 router_tokens_per_expert=[8138, 0, 6662, 0, 1, 0, 0, 3, 0, 0, 0, 8190, 0, 1, 1, 8182, 0, 1, 0, 1, 1, 0, 2, 1, 2, 0, 1, 1, 8191, 8190, 8190, 1, 0, 0, 0, 8191, 8189, 0, 0, 1, 8191, 0, 2, 0, 8191, 0, 7759, 1, 0, 1, 2, 3, 1, 1302, 8191, 0, 716, 1, 0, 8190, 8189, 0, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16176, 16384, 16384] dup_rows_per_dst=[23174, 16364, 16390, 11742, 16378, 10556, 18870, 17808] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16176, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=116452 dedup_ratio=0.500084 router_tokens_per_expert=[7833, 1, 6729, 0, 2, 0, 1, 9, 0, 0, 1, 8188, 1, 2, 3, 8107, 0, 3, 0, 3, 0, 0, 2, 2, 13, 0, 2, 2, 8189, 8189, 8189, 3, 0, 0, 0, 8189, 8185, 0, 2, 3, 8189, 0, 3, 0, 8190, 0, 7319, 2, 0, 3, 16, 8, 3, 1515, 8189, 4, 1209, 2, 3, 8187, 8189, 0, 0, 8188] payload_rows_per_dst=[16366, 16382, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[12784, 16222, 16388, 16412, 16374, 14642, 19470, 18802] combine_rows_per_src=[16366, 16382, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130850 payload_rows_remote=114466 dup_rows=131294 dup_rows_remote=114874 dedup_ratio=0.500847 router_tokens_per_expert=[6679, 4, 6390, 0, 7, 0, 0, 8, 2, 0, 1, 8181, 0, 7, 7, 8180, 0, 7, 0, 10, 0, 10, 7, 2, 21, 0, 5, 6, 8181, 8179, 8184, 10, 0, 0, 0, 8185, 8182, 0, 13, 7, 8185, 1, 5, 0, 8185, 0, 7451, 5, 0, 7, 26, 12, 7, 1737, 8178, 5, 2259, 6, 2, 8176, 8171, 0, 0, 8179] payload_rows_per_dst=[16166, 16382, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[10010, 16374, 16400, 16420, 16390, 14910, 19930, 20860] combine_rows_per_src=[16166, 16382, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=936352 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940460448 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879984544 weight_movement_bytes=176160768 total_nvlink_bytes=2056145312 weight_to_token_ratio=0.093703 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114702 dedup_ratio=0.500000 router_tokens_per_expert=[7956, 0, 6103, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 7880, 0, 0, 0, 6, 0, 0, 1770, 8192, 0, 869, 0, 0, 8192, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[28118, 16370, 16384, 14146, 16384, 20176, 7796, 11698] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=103884 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 6603, 0, 1, 0, 0, 8191, 0, 0, 0, 82, 0, 0, 3636, 8192, 8187, 0, 0, 8192, 0, 8181, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[27188, 16546, 10554, 16384, 12158, 16380, 16384, 15478] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937803776 dispatch_meta_bytes=917480 combine_payload_bytes=937803776 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=938721256 ingress_bytes=1290125312 link_bytes=1290125312 token_bytes=1876525032 weight_movement_bytes=352321536 total_nvlink_bytes=2228846568 weight_to_token_ratio=0.187752 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224163222e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937705472 dispatch_meta_bytes=917360 combine_payload_bytes=937705472 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=938622832 ingress_bytes=1554268160 link_bytes=1554268160 token_bytes=1876328304 weight_movement_bytes=616562688 total_nvlink_bytes=2492890992 weight_to_token_ratio=0.328601 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.703252913e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=924472 combine_payload_bytes=939343872 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940268344 ingress_bytes=1555906560 link_bytes=1555906560 token_bytes=1879612216 weight_movement_bytes=616562688 total_nvlink_bytes=2496174904 weight_to_token_ratio=0.328027 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.712427269e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130874 payload_rows_remote=114490 dup_rows=131270 dup_rows_remote=110136 dedup_ratio=0.500755 router_tokens_per_expert=[7019, 0, 6594, 0, 5, 0, 0, 17, 2, 0, 0, 8184, 0, 5, 7, 8144, 0, 5, 0, 7, 0, 5, 0, 3, 29, 1, 4, 5, 8184, 8187, 8185, 7, 2, 0, 1, 8185, 8181, 0, 11, 5, 8187, 0, 10, 0, 8187, 0, 7829, 4, 0, 7, 0, 9, 5, 732, 8185, 7, 2371, 5, 3, 8184, 8178, 0, 1, 8184] payload_rows_per_dst=[16194, 16382, 16384, 16378, 16384, 16384, 16384, 16384] dup_rows_per_dst=[11076, 16302, 16362, 8266, 16392, 23860, 17878, 21134] combine_rows_per_src=[16194, 16382, 16384, 16378, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917560 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441656 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965752 weight_movement_bytes=176160768 total_nvlink_bytes=2056126520 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=874288 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940398384 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879922480 weight_movement_bytes=176160768 total_nvlink_bytes=2056083248 weight_to_token_ratio=0.093706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937902080 dispatch_meta_bytes=898504 combine_payload_bytes=937902080 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938800584 ingress_bytes=1466384384 link_bytes=1466384384 token_bytes=1876702664 weight_movement_bytes=528482304 total_nvlink_bytes=2405184968 weight_to_token_ratio=0.281602 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.211140451e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130958 payload_rows_remote=114574 dup_rows=131186 dup_rows_remote=114802 dedup_ratio=0.500435 router_tokens_per_expert=[7839, 0, 5041, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8181, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 7732, 0, 0, 0, 44, 5, 0, 2634, 8192, 0, 1323, 0, 0, 8192, 8192, 0, 0, 8192] payload_rows_per_dst=[16270, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[9490, 16322, 16384, 16362, 16384, 15464, 21750, 19030] combine_rows_per_src=[16270, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131018 payload_rows_remote=114634 dup_rows=131126 dup_rows_remote=112236 dedup_ratio=0.500206 router_tokens_per_expert=[7762, 0, 6166, 0, 0, 0, 7, 0, 0, 0, 7, 8192, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8180, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 7459, 0, 0, 0, 13, 0, 0, 1240, 8192, 0, 1958, 0, 0, 8192, 8192, 0, 0, 8192] payload_rows_per_dst=[16330, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[11540, 16350, 16384, 15260, 16384, 16018, 18890, 20300] combine_rows_per_src=[16330, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8190, 0, 3, 1, 1, 0, 1, 8191, 1, 0, 1, 0, 8188, 8191, 39, 8191, 0, 1, 1, 1, 0, 1, 0, 2, 1, 1, 0, 1, 8191, 0, 8151, 0, 1, 1, 8191, 1, 0, 0, 8190, 8191, 0, 1, 8189, 0, 1, 0, 0, 8191, 2, 0, 0, 0, 8191, 8191, 0, 0, 1, 0, 8190, 8191, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16390, 16456, 16388, 16310, 16384, 16380, 16384, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114672 dup_rows=131090 dup_rows_remote=114778 dedup_ratio=0.500069 router_tokens_per_expert=[22, 8184, 16, 0, 11, 8, 1, 6, 8178, 9, 0, 7, 0, 8185, 8186, 33, 8184, 3, 7, 7, 9, 0, 6, 5, 0, 5, 8, 0, 6, 8178, 0, 8150, 4, 4, 0, 8186, 6, 0, 0, 8174, 8185, 0, 4, 8175, 0, 7, 2, 0, 8181, 12, 0, 11, 0, 8160, 8185, 0, 0, 0, 0, 8178, 8184, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16368] dup_rows_per_dst=[16474, 16440, 16430, 16312, 16364, 16362, 16352, 16356] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114678 dup_rows=131086 dup_rows_remote=114766 dedup_ratio=0.500053 router_tokens_per_expert=[3, 8188, 4, 2, 3, 2, 0, 2, 8182, 0, 0, 2, 0, 8186, 8189, 34, 8189, 1, 3, 3, 3, 0, 2, 2, 1, 2, 3, 2, 2, 8185, 0, 8187, 0, 2, 0, 8185, 2, 2, 5, 8188, 8188, 0, 1, 8156, 0, 2, 0, 3, 8188, 3, 1, 5, 0, 8187, 8187, 5, 0, 0, 0, 8187, 8189, 4, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16380, 16382, 16378] dup_rows_per_dst=[16404, 16426, 16400, 16380, 16384, 16320, 16390, 16382] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16380, 16382, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938590208 dispatch_meta_bytes=917504 combine_payload_bytes=938590208 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939507712 ingress_bytes=1114750976 link_bytes=1114750976 token_bytes=1878097920 weight_movement_bytes=176160768 total_nvlink_bytes=2054258688 weight_to_token_ratio=0.093797 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.242140146e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=105598 dedup_ratio=0.500000 router_tokens_per_expert=[8045, 0, 4692, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 7887, 0, 0, 0, 9, 3, 0, 3232, 8192, 0, 720, 0, 0, 8192, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[25474, 16774, 16384, 32472, 16386, 15654, 6488, 1440] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114662 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 8192, 14, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8178, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 1, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16410, 16384, 16356, 16384, 16384, 16386, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939081728 dispatch_meta_bytes=907480 combine_payload_bytes=939081728 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939989208 ingress_bytes=1115242496 link_bytes=1115242496 token_bytes=1879070936 weight_movement_bytes=176160768 total_nvlink_bytes=2055231704 weight_to_token_ratio=0.093749 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244892452e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441584 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965680 weight_movement_bytes=176160768 total_nvlink_bytes=2056126448 weight_to_token_ratio=0.093704 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917800 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940310824 ingress_bytes=1115553792 link_bytes=1115553792 token_bytes=1879703848 weight_movement_bytes=176160768 total_nvlink_bytes=2055864616 weight_to_token_ratio=0.093717 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246635580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917776 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940359952 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879802128 weight_movement_bytes=176160768 total_nvlink_bytes=2055962896 weight_to_token_ratio=0.093712 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130958 payload_rows_remote=114588 dup_rows=131186 dup_rows_remote=114752 dedup_ratio=0.500435 router_tokens_per_expert=[2, 8184, 7, 0, 8, 5, 0, 5, 8180, 7, 0, 5, 0, 8187, 8187, 0, 8186, 2, 7, 5, 6, 0, 5, 3, 0, 4, 6, 0, 5, 8179, 0, 8186, 5, 4, 0, 8187, 5, 0, 0, 8183, 8187, 0, 5, 8180, 0, 5, 0, 4, 8182, 9, 0, 8, 1, 8182, 8187, 0, 0, 0, 0, 8182, 8185, 0, 0, 0] payload_rows_per_dst=[16384, 16284, 16384, 16384, 16384, 16384, 16384, 16370] dup_rows_per_dst=[16258, 16026, 16380, 16752, 16422, 16386, 16528, 16434] combine_rows_per_src=[16384, 16284, 16384, 16384, 16384, 16384, 16384, 16370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 2, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=881144 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940405240 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879929336 weight_movement_bytes=176160768 total_nvlink_bytes=2056090104 weight_to_token_ratio=0.093706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=30720 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917400 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=704643072 egress_bytes=940441496 ingress_bytes=1732247552 link_bytes=1732247552 token_bytes=1879965592 weight_movement_bytes=792723456 total_nvlink_bytes=2672689048 weight_to_token_ratio=0.421669 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699863215e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938704896 dispatch_meta_bytes=917360 combine_payload_bytes=938704896 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939622256 ingress_bytes=1114865664 link_bytes=1114865664 token_bytes=1878327152 weight_movement_bytes=176160768 total_nvlink_bytes=2054487920 weight_to_token_ratio=0.093786 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.242782351e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 53, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8139, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16490, 16384, 16278, 16384, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114690 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8190, 1, 1, 1, 0, 0, 8190, 2, 1, 0, 8191, 0, 0, 1, 1, 0, 8190, 1, 1, 0, 1, 0, 8191, 1, 1, 0, 0, 1, 29, 0, 8191, 5, 0, 8190, 0, 8191, 0, 0, 1, 1, 8191, 0, 8191, 0, 8191, 0, 8191, 8190, 8191, 8158, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 8188] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16382, 16390, 16386, 16444, 16390, 16382, 16320, 16382] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114242 dedup_ratio=0.500061 router_tokens_per_expert=[0, 8181, 11, 7, 9, 0, 24, 8181, 1, 9, 0, 8177, 0, 0, 6, 6, 0, 8182, 0, 5, 0, 53, 0, 8184, 7, 4, 7, 0, 5, 217, 7, 8185, 0, 1, 8162, 0, 8183, 0, 0, 8, 9, 8184, 0, 8178, 5, 8184, 0, 8183, 8183, 8183, 7925, 1, 0, 0, 6, 1, 0, 8, 9, 0, 8, 7, 0, 8176] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16382, 16384, 16380, 16384] dup_rows_per_dst=[16442, 16382, 16474, 16846, 16326, 16366, 15852, 16400] combine_rows_per_src=[16384, 16384, 16374, 16384, 16382, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114680 dedup_ratio=0.500038 router_tokens_per_expert=[0, 8192, 0, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 25, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8171, 0, 0, 0, 8185, 0, 0, 0, 8192, 8187, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8189, 12, 0, 0, 2, 8192, 8192, 6, 0, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16384] dup_rows_per_dst=[16394, 16434, 16384, 16342, 16370, 16352, 16402, 16404] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114716 dedup_ratio=0.500031 router_tokens_per_expert=[1, 8188, 4, 2, 2, 1, 18, 8178, 4, 1, 0, 8186, 0, 0, 2, 2, 0, 8189, 1, 2, 0, 17, 0, 8189, 9, 1, 2, 0, 2, 30, 0, 8183, 1, 3, 8181, 0, 8188, 1, 0, 3, 3, 8189, 3, 8179, 0, 8189, 0, 8189, 8188, 8189, 8134, 2, 1, 13, 2, 3, 0, 3, 3, 0, 2, 3, 0, 8186] payload_rows_per_dst=[16384, 16384, 16380, 16382, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16404, 16384, 16416, 16450, 16370, 16364, 16304, 16388] combine_rows_per_src=[16384, 16384, 16380, 16382, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 8192, 22, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8172, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16492, 16322, 16344, 16380, 16518, 16248, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 1, 0, 0, 0, 8192, 0, 8190, 0, 8192, 0, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16384, 16384, 16390, 16386, 16380, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=264241152 egress_bytes=940425216 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932928 weight_movement_bytes=264241152 total_nvlink_bytes=2144174080 weight_to_token_ratio=0.140559 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=915656 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940308680 ingress_bytes=1115553792 link_bytes=1115553792 token_bytes=1879701704 weight_movement_bytes=176160768 total_nvlink_bytes=2055862472 weight_to_token_ratio=0.093717 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246635580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131032 payload_rows_remote=114648 dup_rows=131112 dup_rows_remote=114712 dedup_ratio=0.500153 router_tokens_per_expert=[0, 8185, 8, 7, 5, 0, 13, 8188, 8, 7, 0, 8181, 0, 0, 5, 4, 0, 8180, 2, 5, 0, 45, 0, 8185, 0, 3, 0, 0, 4, 128, 3, 8187, 1, 2, 8174, 0, 8184, 0, 0, 5, 7, 8185, 0, 8187, 1, 8185, 0, 8183, 8185, 8186, 8015, 4, 0, 0, 5, 3, 0, 7, 7, 0, 5, 6, 2, 8180] payload_rows_per_dst=[16366, 16384, 16366, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16396, 16380, 16294, 15610, 16384, 16498, 17150, 16400] combine_rows_per_src=[16366, 16384, 16366, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917432 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940359608 ingress_bytes=1379844096 link_bytes=1379844096 token_bytes=1879801784 weight_movement_bytes=440401920 total_nvlink_bytes=2320203704 weight_to_token_ratio=0.234281 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726550962e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917584 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940376144 ingress_bytes=1467940864 link_bytes=1467940864 token_bytes=1879834704 weight_movement_bytes=528482304 total_nvlink_bytes=2408317008 weight_to_token_ratio=0.281132 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219856090e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=917440 combine_payload_bytes=939196416 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940113856 ingress_bytes=1115357184 link_bytes=1115357184 token_bytes=1879310272 weight_movement_bytes=176160768 total_nvlink_bytes=2055471040 weight_to_token_ratio=0.093737 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245534657e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 26, 0, 30, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 8166, 0, 8192, 0, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16324, 16384, 16384, 16496, 16384, 16332, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114694 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 2, 2, 0, 1, 0, 0, 52, 0, 1, 0, 0, 1, 8191, 1, 8190, 8191, 0, 1, 0, 1, 8189, 1, 8191, 8186, 1, 0, 1, 0, 8191, 8190, 1, 8190, 8187, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 2, 1, 8190, 1, 1, 1, 8191, 1, 0, 1, 8191, 0, 7, 0, 8191, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16382, 16384, 16384] dup_rows_per_dst=[16388, 16486, 16382, 16376, 16376, 16278, 16392, 16398] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=114730 dedup_ratio=0.500099 router_tokens_per_expert=[0, 0, 0, 12, 8, 0, 6, 1, 0, 29, 0, 7, 0, 0, 5, 8183, 9, 8180, 8170, 0, 4, 0, 5, 8174, 9, 8183, 8167, 5, 1, 11, 0, 8183, 8182, 8, 8163, 8175, 4, 8183, 0, 0, 0, 0, 0, 0, 16, 8173, 5, 14, 5, 9, 8181, 10, 1, 2, 8183, 8, 0, 5, 8184, 13, 6, 0, 8185, 5] payload_rows_per_dst=[16384, 16382, 16368, 16384, 16384, 16376, 16384, 16384] dup_rows_per_dst=[16400, 16426, 16356, 16368, 16316, 16406, 16414, 16412] combine_rows_per_src=[16384, 16382, 16368, 16384, 16384, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114672 dedup_ratio=0.500000 router_tokens_per_expert=[4, 8192, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 20, 0, 12, 0, 0, 0, 0, 8182, 0, 0, 8192, 0, 8192, 3, 0, 0, 0, 8192, 0, 8173, 0, 8192, 0, 8192, 8192, 8192, 8188, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16356, 16384, 16384, 16428, 16390, 16346, 16400, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114684 dup_rows=131080 dup_rows_remote=114734 dedup_ratio=0.500031 router_tokens_per_expert=[1, 0, 0, 4, 3, 0, 2, 1, 0, 58, 0, 3, 0, 0, 5, 8186, 3, 8188, 8161, 0, 0, 0, 0, 8186, 2, 8189, 8187, 1, 1, 4, 0, 8189, 8188, 3, 8184, 8177, 2, 8189, 0, 0, 0, 2, 13, 0, 0, 8158, 0, 1, 2, 2, 8188, 4, 2, 2, 8189, 3, 0, 2, 8189, 3, 4, 2, 8187, 2] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[16392, 16496, 16320, 16384, 16348, 16346, 16400, 16394] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114666 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 5, 0, 7, 0, 0, 1, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 8185, 0, 8192, 0, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16406, 16384, 16384, 16410, 16448, 16404, 16286, 16350] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=917568 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940228672 ingress_bytes=1467793408 link_bytes=1467793408 token_bytes=1879539776 weight_movement_bytes=528482304 total_nvlink_bytes=2408022080 weight_to_token_ratio=0.281176 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219030398e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114668 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8192, 0, 8192, 8189, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8189, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16404, 16378, 16384, 16378, 16374, 16384, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917512 combine_payload_bytes=939491328 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940408840 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879900168 weight_movement_bytes=528482304 total_nvlink_bytes=2408382472 weight_to_token_ratio=0.281123 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114606 dedup_ratio=0.500084 router_tokens_per_expert=[0, 0, 0, 8, 6, 0, 5, 1, 1, 163, 1, 6, 0, 0, 11, 8186, 7, 8184, 8021, 0, 0, 0, 5, 8177, 4, 8185, 8177, 4, 0, 8, 0, 8184, 8183, 7, 8175, 8186, 4, 8185, 0, 0, 0, 1, 0, 0, 11, 8185, 0, 6, 3, 5, 8183, 8, 1, 4, 8185, 6, 0, 5, 8185, 8, 3, 0, 8186, 3] payload_rows_per_dst=[16384, 16384, 16370, 16384, 16384, 16376, 16384, 16384] dup_rows_per_dst=[16356, 15730, 16476, 16408, 16396, 16882, 16358, 16488] combine_rows_per_src=[16384, 16384, 16370, 16384, 16384, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917440 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441536 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965632 weight_movement_bytes=528482304 total_nvlink_bytes=2408447936 weight_to_token_ratio=0.281113 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917672 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940409000 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900328 weight_movement_bytes=352321536 total_nvlink_bytes=2232221864 weight_to_token_ratio=0.187415 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917416 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940441512 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965608 weight_movement_bytes=264241152 total_nvlink_bytes=2144206760 weight_to_token_ratio=0.140556 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917424 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441520 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965616 weight_movement_bytes=176160768 total_nvlink_bytes=2056126384 weight_to_token_ratio=0.093704 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.024621 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917088 combine_payload_bytes=939343872 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940260960 ingress_bytes=1291665408 link_bytes=1291665408 token_bytes=1879604832 weight_movement_bytes=352321536 total_nvlink_bytes=2231926368 weight_to_token_ratio=0.187444 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232787117e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114716 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 8192, 0, 8192, 8191, 0, 6, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 13, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16608, 16394, 16384, 16356, 16152, 16384, 16410] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=114616 dedup_ratio=0.500092 router_tokens_per_expert=[7, 0, 8184, 8, 6, 1, 11, 0, 0, 8181, 9, 0, 8177, 17, 0, 8173, 12, 0, 4, 8174, 8, 8, 8180, 11, 18, 10, 8173, 0, 0, 0, 6, 49, 0, 32, 8180, 19, 0, 5, 8162, 0, 14, 6, 8177, 0, 9, 8137, 4, 9, 2, 0, 0, 2, 15, 0, 0, 0, 10, 0, 8182, 8183, 8168, 8147, 8176, 6] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16376, 16384, 16370, 16382] dup_rows_per_dst=[16412, 16368, 16410, 16480, 16420, 16328, 16398, 16280] combine_rows_per_src=[16384, 16384, 16384, 16384, 16376, 16384, 16370, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114688 dedup_ratio=0.500015 router_tokens_per_expert=[1, 0, 8191, 0, 1, 0, 1, 0, 0, 8190, 1, 0, 8191, 3, 0, 8190, 1, 0, 1, 8190, 1, 1, 8190, 2, 0, 2, 8191, 0, 2, 0, 1, 0, 0, 3, 8190, 0, 0, 1, 8191, 0, 2, 1, 8190, 0, 1, 8191, 0, 2, 0, 1, 0, 0, 2, 0, 0, 0, 1, 0, 8191, 8189, 8189, 8186, 8190, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16384, 16386, 16388, 16388, 16386, 16390, 16382, 16372] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917616 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441712 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965808 weight_movement_bytes=528482304 total_nvlink_bytes=2408448112 weight_to_token_ratio=0.281113 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 8189, 0, 8192, 8189, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8158, 0, 8192, 5, 0, 0, 0, 24, 0, 0, 8155, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 7, 0, 8185, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16474, 16378, 16384, 16326, 16358, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=917120 combine_payload_bytes=939327488 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940244608 ingress_bytes=1115488256 link_bytes=1115488256 token_bytes=1879572096 weight_movement_bytes=176160768 total_nvlink_bytes=2055732864 weight_to_token_ratio=0.093724 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246268606e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114718 dedup_ratio=0.500008 router_tokens_per_expert=[2, 1, 8188, 1, 2, 1, 4, 0, 0, 8188, 3, 0, 8187, 4, 0, 8185, 4, 1, 2, 8185, 3, 2, 8188, 4, 3, 3, 8187, 0, 0, 0, 2, 13, 0, 42, 8188, 12, 0, 2, 8177, 0, 4, 2, 8182, 0, 3, 8176, 0, 3, 0, 3, 1, 7, 6, 2, 1, 0, 3, 0, 8189, 8188, 8184, 8147, 8187, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16390, 16374, 16394, 16406, 16460, 16356, 16410, 16284] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114692 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 8192, 0, 8192, 8191, 0, 11, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8183, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 9, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16384, 16450, 16422, 16384, 16578, 16298, 16356, 16204] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917488 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940408816 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900144 weight_movement_bytes=352321536 total_nvlink_bytes=2232221680 weight_to_token_ratio=0.187415 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 9, 8192, 2, 0, 0, 8190, 0, 0, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 8192, 8192, 8184, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16384, 16402, 16380, 16386, 16368] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114680 dup_rows=131084 dup_rows_remote=114386 dedup_ratio=0.500046 router_tokens_per_expert=[5, 0, 8185, 2, 4, 2, 8, 0, 0, 8184, 7, 0, 8181, 14, 0, 8177, 9, 1, 4, 8182, 6, 5, 8184, 8, 7, 7, 8182, 0, 0, 0, 5, 20, 2, 0, 8184, 0, 0, 3, 8185, 0, 8, 4, 8186, 0, 7, 8164, 3, 8, 0, 7, 0, 0, 12, 0, 0, 0, 6, 0, 8185, 8184, 8172, 8184, 8179, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16376, 16384, 16384, 16380] dup_rows_per_dst=[16362, 16402, 16308, 16256, 16196, 16550, 16312, 16698] combine_rows_per_src=[16384, 16384, 16384, 16384, 16376, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917616 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425328 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879933040 weight_movement_bytes=176160768 total_nvlink_bytes=2056093808 weight_to_token_ratio=0.093706 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917504 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940408832 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900160 weight_movement_bytes=352321536 total_nvlink_bytes=2232221696 weight_to_token_ratio=0.187415 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114650 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 50, 8192, 21, 0, 0, 8140, 0, 0, 0, 8171, 0, 0, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 8192, 8192, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16384, 16422, 16342, 16388, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=916264 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940374824 ingress_bytes=1644101632 link_bytes=1644101632 token_bytes=1879833384 weight_movement_bytes=704643072 total_nvlink_bytes=2584476456 weight_to_token_ratio=0.374843 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206282857e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130700 payload_rows_remote=114318 dup_rows=131444 dup_rows_remote=115072 dedup_ratio=0.501419 router_tokens_per_expert=[0, 8182, 2, 8170, 63, 8182, 0, 8089, 0, 13, 0, 6305, 10, 8173, 0, 0, 8180, 7852, 29, 0, 0, 0, 8184, 0, 0, 8, 8154, 0, 4, 6, 15, 8, 0, 9, 8, 8184, 27, 8163, 5, 8, 14, 8168, 7, 9, 11, 0, 0, 8185, 3, 6, 8180, 4, 7, 5, 15, 1484, 10, 876, 0, 1, 0, 8053, 1, 0] payload_rows_per_dst=[16372, 16384, 16026, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16276, 13010, 15704, 16372, 16424, 16404, 19392, 17862] combine_rows_per_src=[16372, 16384, 16026, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131006 payload_rows_remote=114688 dup_rows=131138 dup_rows_remote=129774 dedup_ratio=0.500252 router_tokens_per_expert=[2, 8191, 1, 8190, 8, 8190, 0, 8183, 0, 1, 0, 6362, 1, 8186, 0, 0, 8191, 8158, 0, 0, 1, 0, 8191, 0, 0, 1, 8189, 0, 0, 1, 0, 1, 0, 1, 1, 8191, 2, 8189, 1, 1, 9, 8190, 1, 1, 2, 0, 0, 8191, 0, 125, 8191, 1, 1, 0, 1, 577, 2, 1156, 0, 0, 0, 8190, 0, 0] payload_rows_per_dst=[16384, 16384, 16318, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 27734, 1364, 16382, 16388, 16404, 17790, 18692] combine_rows_per_src=[16384, 16384, 16318, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917352 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940441448 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965544 weight_movement_bytes=528482304 total_nvlink_bytes=2408447848 weight_to_token_ratio=0.281113 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=936493056 dispatch_meta_bytes=917560 combine_payload_bytes=936493056 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=937410616 ingress_bytes=1200734208 link_bytes=1200734208 token_bytes=1873903672 weight_movement_bytes=264241152 total_nvlink_bytes=2138144824 weight_to_token_ratio=0.141011 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.723610354e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114652 dedup_ratio=0.500000 router_tokens_per_expert=[0, 1, 8185, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 8192, 1, 2, 0, 0, 0, 0, 32, 8192, 32, 0, 0, 8160, 0, 0, 0, 8178, 0, 0, 8192, 0, 0, 1, 0, 3, 14, 0, 0, 0, 0, 0, 0, 8192, 8192, 8192, 8160, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16372, 16384, 16382, 16390, 16448, 16356, 16420, 16320] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114636 dedup_ratio=0.500031 router_tokens_per_expert=[0, 8187, 2, 8184, 25, 8188, 0, 8159, 0, 2, 0, 5774, 3, 8183, 0, 0, 8179, 8122, 12, 5, 0, 0, 8189, 2, 0, 3, 8179, 0, 5, 3, 10, 3, 34, 3, 2, 8184, 8, 8179, 2, 3, 28, 8187, 2, 3, 4, 0, 1, 8189, 1, 38, 8188, 1, 3, 2, 6, 722, 4, 1720, 1, 0, 0, 8137, 1, 0] payload_rows_per_dst=[16382, 16382, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16358, 11542, 16260, 16396, 16446, 16444, 17914, 19720] combine_rows_per_src=[16382, 16382, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 8192, 12, 0, 0, 8184, 0, 0, 0, 8185, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 8192, 8192, 8188, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16390, 16422, 16384, 16356, 16370, 16390, 16376] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=977848 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940501944 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1880026040 weight_movement_bytes=528482304 total_nvlink_bytes=2408508344 weight_to_token_ratio=0.281104 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=106864 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 3912, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7, 0, 0, 8192, 0, 8192, 0, 0, 14, 8192, 0, 0, 0, 0, 0, 8192, 0, 45, 8192, 0, 0, 0, 0, 2691, 0, 1523, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[20114, 24208, 12756, 16384, 16398, 16412, 14226, 10574] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130980 payload_rows_remote=114596 dup_rows=131164 dup_rows_remote=112044 dedup_ratio=0.500351 router_tokens_per_expert=[0, 8184, 5, 8179, 41, 8184, 0, 8110, 1, 6, 0, 6203, 6, 8179, 0, 0, 8182, 7963, 30, 0, 0, 0, 8185, 3, 0, 7, 8162, 0, 0, 7, 8, 6, 0, 8, 5, 8187, 18, 8170, 3, 6, 4, 8175, 5, 6, 8, 0, 0, 8186, 1, 18, 8184, 4, 6, 2, 10, 848, 8, 1472, 0, 0, 2, 8083, 2, 0] payload_rows_per_dst=[16304, 16384, 16372, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 12406, 32342, 16372, 16246, 16548, 18130, 19120] combine_rows_per_src=[16304, 16384, 16372, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917360 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441456 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965552 weight_movement_bytes=352321536 total_nvlink_bytes=2232287088 weight_to_token_ratio=0.187409 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917264 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940375824 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879834384 weight_movement_bytes=352321536 total_nvlink_bytes=2232155920 weight_to_token_ratio=0.187422 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 8192, 0, 8192, 0, 0, 1, 4959, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 8192, 0, 0, 6, 8192, 0, 0, 0, 0, 0, 8192, 0, 227, 8192, 0, 0, 0, 0, 1365, 0, 1632, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 9920, 16384, 16384, 16388, 16396, 19568, 19648] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114638 dup_rows=131122 dup_rows_remote=114760 dedup_ratio=0.500191 router_tokens_per_expert=[7185, 0, 8, 6, 0, 0, 234, 8175, 9, 11, 9, 8, 4, 6, 0, 90, 8098, 0, 0, 7, 8173, 765, 8178, 0, 0, 0, 8184, 11, 8181, 0, 8181, 0, 8183, 25, 0, 8184, 0, 8, 5, 0, 0, 57, 0, 0, 8183, 12, 0, 0, 9, 8184, 8183, 8129, 6, 0, 8162, 0, 8, 4, 10, 8, 5, 8175, 0, 9] payload_rows_per_dst=[16376, 16384, 16384, 16384, 16384, 16384, 16342, 16384] dup_rows_per_dst=[14840, 16624, 17702, 16362, 16426, 16488, 16270, 16410] combine_rows_per_src=[16376, 16384, 16384, 16384, 16384, 16384, 16342, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=886208 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940410304 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879934400 weight_movement_bytes=176160768 total_nvlink_bytes=2056095168 weight_to_token_ratio=0.093706 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938770432 dispatch_meta_bytes=906560 combine_payload_bytes=938770432 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939676992 ingress_bytes=1114931200 link_bytes=1114931200 token_bytes=1878447424 weight_movement_bytes=176160768 total_nvlink_bytes=2054608192 weight_to_token_ratio=0.093780 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243149325e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114672 dedup_ratio=0.500023 router_tokens_per_expert=[7641, 0, 2, 0, 0, 0, 537, 8190, 1, 1, 1, 1, 0, 1, 0, 2, 8190, 0, 0, 1, 8190, 14, 8190, 0, 0, 0, 8191, 2, 8191, 0, 8190, 0, 8191, 1, 0, 8191, 1, 1, 0, 0, 0, 1, 0, 0, 8191, 2, 0, 0, 1, 8191, 8190, 8189, 1, 0, 8189, 1, 2, 2, 0, 0, 1, 8189, 0, 1] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[16358, 16392, 16406, 16382, 16386, 16386, 16382, 16386] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940441584 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965680 weight_movement_bytes=176160768 total_nvlink_bytes=2056126448 weight_to_token_ratio=0.093704 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=917592 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940032088 ingress_bytes=1467596800 link_bytes=1467596800 token_bytes=1879146584 weight_movement_bytes=528482304 total_nvlink_bytes=2407628888 weight_to_token_ratio=0.281235 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217929475e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=111016 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 3349, 0, 8192, 5, 0, 8187, 8192, 0, 9, 0, 0, 8192, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 11, 0, 0, 8184, 0, 8192, 0, 0, 36, 8192, 0, 0, 0, 0, 21, 8191, 0, 712, 8192, 0, 0, 0, 0, 1124, 0, 2931, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[12458, 6708, 20318, 16400, 16390, 16496, 20056, 22246] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=97880 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 4581, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 4, 0, 0, 8192, 0, 8192, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 8192, 0, 183, 8192, 0, 0, 0, 0, 1815, 0, 1605, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[33192, 25546, 16384, 16384, 16392, 15968, 3996, 3210] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=114686 dedup_ratio=0.500107 router_tokens_per_expert=[7856, 1, 0, 0, 0, 0, 147, 8186, 3, 3, 3, 3, 2, 2, 0, 21, 8182, 0, 0, 2, 8156, 199, 8187, 0, 1, 0, 8189, 4, 8188, 0, 8189, 5, 8189, 6, 1, 8188, 1, 3, 6, 0, 1, 21, 0, 0, 8184, 4, 0, 0, 3, 8189, 8189, 8171, 3, 0, 8181, 1, 3, 1, 4, 1, 1, 8186, 3, 3] payload_rows_per_dst=[16372, 16384, 16384, 16384, 16384, 16384, 16368, 16384] dup_rows_per_dst=[16008, 16446, 16694, 16390, 16404, 16414, 16350, 16394] combine_rows_per_src=[16372, 16384, 16384, 16384, 16384, 16384, 16368, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=917416 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940392360 ingress_bytes=1467957248 link_bytes=1467957248 token_bytes=1879867304 weight_movement_bytes=528482304 total_nvlink_bytes=2408349608 weight_to_token_ratio=0.281127 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219947833e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 2, 3, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8100, 0, 0, 0, 0, 0, 56, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 36, 8192, 1, 0, 0, 8192, 0, 8192, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16312, 16384, 16458, 16384, 16384, 16382, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114640 dup_rows=131120 dup_rows_remote=114716 dedup_ratio=0.500183 router_tokens_per_expert=[7491, 0, 7, 0, 0, 0, 394, 8179, 7, 8, 8, 7, 5, 5, 0, 57, 8169, 0, 0, 5, 8135, 301, 8180, 0, 2, 0, 8186, 8, 8184, 0, 8184, 0, 8185, 15, 0, 8185, 3, 7, 6, 0, 2, 33, 0, 0, 8184, 10, 0, 0, 7, 8185, 8183, 8154, 5, 0, 8171, 1, 6, 2, 8, 3, 4, 8179, 5, 7] payload_rows_per_dst=[16364, 16384, 16384, 16384, 16384, 16384, 16356, 16384] dup_rows_per_dst=[15778, 16074, 16836, 16128, 16418, 16446, 17036, 16404] combine_rows_per_src=[16364, 16384, 16384, 16384, 16384, 16384, 16356, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=902816 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940426912 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879951008 weight_movement_bytes=176160768 total_nvlink_bytes=2056111776 weight_to_token_ratio=0.093705 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=850272 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=940374368 ingress_bytes=1732247552 link_bytes=1732247552 token_bytes=1879898464 weight_movement_bytes=792723456 total_nvlink_bytes=2672621920 weight_to_token_ratio=0.421684 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699863215e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=917384 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940212104 ingress_bytes=1115455488 link_bytes=1115455488 token_bytes=1879506824 weight_movement_bytes=176160768 total_nvlink_bytes=2055667592 weight_to_token_ratio=0.093727 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246085119e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[7759, 0, 0, 0, 0, 0, 345, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 88, 8192, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16208, 16384, 16560, 16384, 16384, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114688 dup_rows=131078 dup_rows_remote=114588 dedup_ratio=0.500023 router_tokens_per_expert=[10, 9, 7, 0, 5, 73, 0, 0, 40, 3, 10, 0, 11, 7, 0, 8179, 8174, 0, 8183, 8177, 6, 8181, 0, 18, 8180, 8, 8183, 0, 0, 101, 8145, 8183, 0, 7, 0, 8084, 9, 8184, 0, 8113, 10, 0, 9, 0, 8167, 2, 1, 2, 13, 3, 0, 0, 1, 11, 8182, 0, 0, 8170, 8, 8181, 1, 0, 12, 9] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16556, 16484, 16366, 16490, 16042, 16364, 16398, 16378] combine_rows_per_src=[16384, 16384, 16384, 16378, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[4, 0, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=917424 combine_payload_bytes=939130880 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940048304 ingress_bytes=1115291648 link_bytes=1115291648 token_bytes=1879179184 weight_movement_bytes=176160768 total_nvlink_bytes=2055339952 weight_to_token_ratio=0.093743 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245167683e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[2, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 2, 1, 0, 8190, 8189, 1, 8191, 8190, 1, 8190, 0, 1, 8190, 2, 8190, 0, 0, 80, 8189, 8191, 0, 1, 0, 8111, 1, 8190, 0, 8190, 1, 2, 1, 0, 8190, 1, 0, 0, 2, 0, 0, 0, 0, 2, 8190, 0, 0, 8190, 1, 8191, 0, 0, 2, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16386, 16380, 16538, 16222, 16388, 16384, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917104 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940441200 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879965296 weight_movement_bytes=880803840 total_nvlink_bytes=2760769136 weight_to_token_ratio=0.468521 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940441616 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965712 weight_movement_bytes=704643072 total_nvlink_bytes=2584608784 weight_to_token_ratio=0.374817 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=111832 dedup_ratio=0.500008 router_tokens_per_expert=[8045, 0, 0, 0, 0, 0, 145, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 2, 8192, 1, 0, 0, 8192, 0, 8192, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[19242, 16384, 14014, 16656, 16284, 16212, 15972, 16310] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8050, 0, 0, 0, 0, 0, 134, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 8, 8192, 5, 0, 0, 8192, 0, 8188, 0, 8192, 5, 8192, 0, 2, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16368, 16384, 16402, 16386, 16386, 16362, 16384, 16400] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114700 dedup_ratio=0.500015 router_tokens_per_expert=[4, 3, 2, 0, 1, 16, 0, 0, 12, 2, 6, 0, 4, 3, 0, 8185, 8188, 0, 8189, 8186, 2, 8188, 0, 5, 8185, 3, 8189, 0, 0, 50, 8176, 8179, 0, 3, 0, 8140, 10, 8184, 0, 8174, 3, 0, 3, 0, 8182, 1, 1, 1, 4, 1, 0, 0, 0, 4, 8188, 7, 0, 8187, 3, 8188, 2, 0, 5, 3] payload_rows_per_dst=[16384, 16384, 16382, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16404, 16408, 16380, 16446, 16270, 16376, 16400, 16392] combine_rows_per_src=[16384, 16384, 16382, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29987133880 prefetch_bytes=2554331136 reduce_bytes=2994733056 weight_movement_bytes=5549064192 egress_bytes=15000885688 ingress_bytes=20535312384 link_bytes=20535312384 total_nvlink_bytes=35536198072 weight_to_token_ratio=0.185048 prefetch_share_of_ingress=0.124387 dedup_ratio=0.501493 prefetch_slots=58 unused_prefetch_slots=454 slot_utilization=0.113281 reduce_slots=34 remote_demand=29 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020656 gemm_inner_pad_upper_bound=34036 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.186823593e-05 prefetch_bytes_per_gemm_flop=8.939492582e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 4, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29952659360 prefetch_bytes=2730491904 reduce_bytes=3170893824 weight_movement_bytes=5901385728 egress_bytes=14983696288 ingress_bytes=20870348800 link_bytes=20870348800 total_nvlink_bytes=35854045088 weight_to_token_ratio=0.197024 prefetch_share_of_ingress=0.130831 dedup_ratio=0.502127 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=36 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020715 gemm_inner_pad_upper_bound=34163 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.304077598e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 8192, 8192, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 27, 8192, 8192, 0, 0, 0, 8165, 1, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16382, 16436, 16332, 16384, 16384, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114696 dedup_ratio=0.500031 router_tokens_per_expert=[8, 8, 5, 0, 3, 37, 0, 0, 30, 3, 7, 0, 8, 6, 0, 8182, 8183, 0, 8185, 8180, 5, 8183, 0, 8, 8185, 7, 8184, 0, 0, 0, 8147, 8185, 0, 5, 0, 8187, 0, 8186, 0, 8161, 8, 0, 7, 0, 8174, 3, 4, 1, 9, 2, 0, 0, 0, 8, 8184, 0, 0, 8178, 7, 8184, 0, 0, 10, 5] payload_rows_per_dst=[16384, 16384, 16380, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16208, 16272, 16344, 15176, 17900, 16408, 16388, 16384] combine_rows_per_src=[16384, 16384, 16380, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=906072 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940413784 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879921496 weight_movement_bytes=176160768 total_nvlink_bytes=2056082264 weight_to_token_ratio=0.093706 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=115176 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8169, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 267, 8192, 8192, 0, 0, 0, 7925, 23, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16338, 16918, 15896, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940441600 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965696 weight_movement_bytes=704643072 total_nvlink_bytes=2584608768 weight_to_token_ratio=0.374817 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917536 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940408864 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900192 weight_movement_bytes=352321536 total_nvlink_bytes=2232221728 weight_to_token_ratio=0.187415 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917504 combine_payload_bytes=939458560 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940376064 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834624 weight_movement_bytes=176160768 total_nvlink_bytes=2055995392 weight_to_token_ratio=0.093711 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=919456 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940443552 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879967648 weight_movement_bytes=352321536 total_nvlink_bytes=2232289184 weight_to_token_ratio=0.187408 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=30002372456 prefetch_bytes=3258974208 reduce_bytes=2906652672 weight_movement_bytes=6165626880 egress_bytes=15008505704 ingress_bytes=21159493632 link_bytes=21159493632 total_nvlink_bytes=36167999336 weight_to_token_ratio=0.205505 prefetch_share_of_ingress=0.154019 dedup_ratio=0.501304 prefetch_slots=74 unused_prefetch_slots=438 slot_utilization=0.144531 reduce_slots=33 remote_demand=37 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021248 gemm_inner_pad_upper_bound=33147 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.405270746e-05 prefetch_bytes_per_gemm_flop=1.140555950e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29939161504 prefetch_bytes=3082813440 reduce_bytes=1673527296 weight_movement_bytes=4756340736 egress_bytes=14976866720 ingress_bytes=19718635520 link_bytes=19718635520 total_nvlink_bytes=34695502240 weight_to_token_ratio=0.158867 prefetch_share_of_ingress=0.156340 dedup_ratio=0.502121 prefetch_slots=70 unused_prefetch_slots=442 slot_utilization=0.136719 reduce_slots=19 remote_demand=35 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020597 gemm_inner_pad_upper_bound=33909 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.901007997e-05 prefetch_bytes_per_gemm_flop=1.078904277e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=30006678144 prefetch_bytes=2113929216 reduce_bytes=3435134976 weight_movement_bytes=5549064192 egress_bytes=15010697856 ingress_bytes=20545044480 link_bytes=20545044480 total_nvlink_bytes=35555742336 weight_to_token_ratio=0.184928 prefetch_share_of_ingress=0.102892 dedup_ratio=0.501420 prefetch_slots=48 unused_prefetch_slots=464 slot_utilization=0.093750 reduce_slots=39 remote_demand=24 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021070 gemm_inner_pad_upper_bound=33147 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.190229573e-05 prefetch_bytes_per_gemm_flop=7.398200758e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29870793536 prefetch_bytes=2730491904 reduce_bytes=2642411520 weight_movement_bytes=5372903424 egress_bytes=14942692160 ingress_bytes=20301004800 link_bytes=20301004800 total_nvlink_bytes=35243696960 weight_to_token_ratio=0.179871 prefetch_share_of_ingress=0.134500 dedup_ratio=0.503135 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=30 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020715 gemm_inner_pad_upper_bound=33782 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.104822051e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8185, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 50, 8192, 8192, 0, 0, 0, 8143, 5, 8192, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 2, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16504, 16428, 16294, 16384, 16334, 16360] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114672 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 8192, 0, 0, 8176, 0, 8192, 0, 0, 150, 8192, 8182, 0, 0, 0, 8042, 16, 8185, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 8192, 0, 8192, 9, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16364, 16370, 16384, 16652, 16116, 16384, 16400, 16402] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917440 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441536 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965632 weight_movement_bytes=176160768 total_nvlink_bytes=2056126400 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29898224744 prefetch_bytes=3347054592 reduce_bytes=1937768448 weight_movement_bytes=5284823040 egress_bytes=14956344424 ingress_bytes=20226703360 link_bytes=20226703360 total_nvlink_bytes=35183047784 weight_to_token_ratio=0.176760 prefetch_share_of_ingress=0.165477 dedup_ratio=0.502736 prefetch_slots=76 unused_prefetch_slots=436 slot_utilization=0.148438 reduce_slots=22 remote_demand=38 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020241 gemm_inner_pad_upper_bound=34544 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.078818486e-05 prefetch_bytes_per_gemm_flop=1.171381787e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29976437568 prefetch_bytes=2290089984 reduce_bytes=3347054592 weight_movement_bytes=5637144576 egress_bytes=14995530560 ingress_bytes=20618051584 link_bytes=20618051584 total_nvlink_bytes=35613582144 weight_to_token_ratio=0.188053 prefetch_share_of_ingress=0.111072 dedup_ratio=0.501543 prefetch_slots=52 unused_prefetch_slots=460 slot_utilization=0.101562 reduce_slots=38 remote_demand=26 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020064 gemm_inner_pad_upper_bound=35052 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.215780155e-05 prefetch_bytes_per_gemm_flop=8.014717487e-06 anomalies=none - [2026-09-16 21:39:32] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 607.2 | throughput per GPU (TFLOP/s/GPU): 549.1 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.004412E+01 | loss scale: 1.0 | grad norm: 7.308 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123744 payload_rows_remote=107454 dup_rows=138400 dup_rows_remote=114926 dedup_ratio=0.527954 router_tokens_per_expert=[2802, 47, 264, 34, 1452, 0, 5105, 5, 974, 5513, 214, 562, 1981, 3614, 0, 1236, 2993, 1493, 1935, 1220, 2, 1790, 819, 4146, 3, 41, 2485, 3756, 6942, 102, 5750, 3381, 207, 918, 4844, 998, 1261, 22, 7017, 339, 2466, 6, 7330, 4351, 2, 1748, 1356, 1552, 7680, 858, 2423, 1, 1, 341, 65, 2086, 1, 195, 175, 4203, 7184, 3060, 4223, 3498] payload_rows_per_dst=[16158, 14448, 14456, 15588, 15920, 15060, 16290, 15824] dup_rows_per_dst=[17628, 13740, 14340, 15870, 15292, 16528, 23474, 21528] combine_rows_per_src=[16158, 14448, 14456, 15588, 15920, 15060, 16290, 15824] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124130 payload_rows_remote=107842 dup_rows=138014 dup_rows_remote=110632 dedup_ratio=0.526482 router_tokens_per_expert=[3306, 11, 187, 66, 1500, 3, 4155, 15, 1330, 5165, 479, 541, 1635, 4316, 5, 1582, 2724, 1296, 2941, 1752, 4, 2164, 808, 4382, 16, 17, 2247, 3712, 6333, 41, 5854, 2681, 224, 421, 5243, 830, 1339, 71, 6962, 203, 2264, 43, 7160, 4782, 0, 1851, 1347, 1037, 7649, 1287, 3029, 0, 4, 234, 9, 1802, 0, 156, 72, 3853, 7327, 3373, 3214, 4018] payload_rows_per_dst=[16136, 16288, 16210, 15398, 15854, 13390, 15834, 15020] dup_rows_per_dst=[17004, 27382, 28852, 14188, 14732, 9310, 12194, 14352] combine_rows_per_src=[16136, 16288, 16210, 15398, 15854, 13390, 15834, 15020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=124644 payload_rows_remote=108350 dup_rows=137500 dup_rows_remote=111636 dedup_ratio=0.524521 router_tokens_per_expert=[3683, 18, 79, 52, 829, 2, 5428, 6, 1223, 6215, 215, 425, 1699, 4320, 0, 947, 1816, 648, 1801, 1194, 4, 1850, 833, 4288, 1, 11, 2498, 3531, 6938, 115, 5644, 2577, 62, 525, 5143, 1018, 1152, 21, 7007, 458, 3734, 7, 7624, 5299, 0, 1996, 1045, 1567, 7535, 1234, 3253, 0, 0, 372, 2, 1981, 2, 84, 71, 4353, 7226, 1915, 4117, 3379] payload_rows_per_dst=[16228, 15148, 13916, 15742, 16130, 16294, 15978, 15208] dup_rows_per_dst=[18418, 14940, 10952, 14786, 21120, 25864, 14746, 16674] combine_rows_per_src=[16228, 15148, 13916, 15742, 16130, 16294, 15978, 15208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125222 payload_rows_remote=109812 dup_rows=136922 dup_rows_remote=117138 dedup_ratio=0.522316 router_tokens_per_expert=[2403, 10, 118, 8, 1259, 3, 4497, 11, 1537, 5610, 383, 780, 2329, 3136, 0, 1260, 3000, 1522, 1658, 1460, 1, 1933, 636, 4105, 2, 13, 3158, 3714, 6479, 69, 5803, 2484, 106, 696, 5373, 631, 864, 27, 6965, 544, 2819, 29, 7612, 4503, 0, 1721, 1178, 1075, 7549, 1328, 3404, 5, 1, 327, 0, 2375, 0, 211, 78, 4710, 7421, 2270, 3813, 4056] payload_rows_per_dst=[16150, 15278, 15410, 15860, 15814, 15656, 15888, 15166] dup_rows_per_dst=[15310, 20534, 19784, 19172, 14598, 18324, 14090, 15110] combine_rows_per_src=[16150, 15278, 15410, 15860, 15814, 15656, 15888, 15166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122350 payload_rows_remote=106294 dup_rows=139794 dup_rows_remote=110704 dedup_ratio=0.533272 router_tokens_per_expert=[2176, 8, 174, 7, 1145, 0, 5008, 5, 1242, 5693, 236, 529, 2327, 3938, 0, 934, 1859, 1410, 1687, 1501, 2, 1950, 672, 4243, 0, 14, 3084, 4017, 7085, 78, 6112, 3678, 163, 571, 4371, 1056, 1421, 29, 6778, 526, 2498, 2, 7460, 4947, 0, 1836, 1691, 1206, 7260, 1447, 2619, 1, 0, 333, 11, 2493, 5, 88, 40, 4093, 7402, 2361, 4055, 3495] payload_rows_per_dst=[14772, 14732, 13928, 15444, 15848, 15254, 16316, 16056] dup_rows_per_dst=[11038, 15066, 12720, 15562, 13982, 16752, 25584, 29090] combine_rows_per_src=[14772, 14732, 13928, 15444, 15848, 15254, 16316, 16056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123358 payload_rows_remote=107586 dup_rows=138786 dup_rows_remote=118896 dedup_ratio=0.529427 router_tokens_per_expert=[2640, 57, 75, 46, 2081, 2, 4944, 10, 1177, 4000, 283, 425, 2007, 2812, 4, 1313, 2697, 1342, 1678, 1950, 0, 1933, 895, 4068, 29, 13, 2805, 3358, 6191, 35, 5450, 3315, 587, 690, 5614, 678, 847, 54, 6791, 507, 2911, 21, 7383, 4401, 5, 2040, 1542, 1560, 7211, 917, 3422, 1, 4, 331, 10, 2448, 2, 171, 226, 4520, 6704, 2992, 4155, 4692] payload_rows_per_dst=[16082, 13140, 15064, 15772, 15906, 16264, 15552, 15578] dup_rows_per_dst=[17036, 10902, 18494, 19890, 17928, 23462, 13136, 17938] combine_rows_per_src=[16082, 13140, 15064, 15772, 15906, 16264, 15552, 15578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[3, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=880263168 dispatch_meta_bytes=889520 combine_payload_bytes=880263168 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=881152688 ingress_bytes=1056423936 link_bytes=1056423936 token_bytes=1761415856 weight_movement_bytes=176160768 total_nvlink_bytes=1937576624 weight_to_token_ratio=0.100011 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.915533069e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=883441664 dispatch_meta_bytes=873896 combine_payload_bytes=883441664 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=884315560 ingress_bytes=1059602432 link_bytes=1059602432 token_bytes=1767757224 weight_movement_bytes=176160768 total_nvlink_bytes=1943917992 weight_to_token_ratio=0.099652 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=22272 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.933331319e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=887603200 dispatch_meta_bytes=879944 combine_payload_bytes=887603200 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=888483144 ingress_bytes=1416085504 link_bytes=1416085504 token_bytes=1776086344 weight_movement_bytes=528482304 total_nvlink_bytes=2304568648 weight_to_token_ratio=0.297554 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.022727 gemm_active_groups=18 gemm_peak_group_rows=19072 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.929487719e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=899579904 dispatch_meta_bytes=907800 combine_payload_bytes=899579904 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=900487704 ingress_bytes=1075740672 link_bytes=1075740672 token_bytes=1800067608 weight_movement_bytes=176160768 total_nvlink_bytes=1976228376 weight_to_token_ratio=0.097863 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=17152 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.023698727e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123708 payload_rows_remote=107560 dup_rows=138436 dup_rows_remote=115614 dedup_ratio=0.528091 router_tokens_per_expert=[3127, 18, 93, 77, 1328, 0, 5114, 21, 1342, 5228, 356, 714, 1801, 3460, 0, 1210, 1349, 1378, 1605, 2026, 8, 1869, 855, 3996, 0, 32, 2806, 3926, 6896, 75, 5922, 2723, 85, 820, 5534, 795, 1080, 43, 6367, 838, 2483, 25, 7543, 4586, 14, 1617, 1057, 1456, 7627, 1975, 3907, 0, 10, 588, 0, 2391, 1, 98, 39, 4757, 7343, 2315, 2805, 3518] payload_rows_per_dst=[16192, 14460, 14056, 16084, 16148, 16252, 15964, 14552] dup_rows_per_dst=[18050, 13762, 12116, 20830, 22822, 21310, 17032, 12514] combine_rows_per_src=[16192, 14460, 14056, 16084, 16148, 16252, 15964, 14552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=870760448 dispatch_meta_bytes=867992 combine_payload_bytes=870760448 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=871628440 ingress_bytes=1311162368 link_bytes=1311162368 token_bytes=1742388888 weight_movement_bytes=440401920 total_nvlink_bytes=2182790808 weight_to_token_ratio=0.252758 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=18048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.341961955e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=881344512 dispatch_meta_bytes=905928 combine_payload_bytes=881344512 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=882250440 ingress_bytes=1585987584 link_bytes=1585987584 token_bytes=1763594952 weight_movement_bytes=704643072 total_nvlink_bytes=2468238024 weight_to_token_ratio=0.399549 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=15360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.880868447e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123872 payload_rows_remote=107712 dup_rows=138272 dup_rows_remote=118490 dedup_ratio=0.527466 router_tokens_per_expert=[2832, 41, 176, 40, 839, 0, 5115, 14, 888, 5477, 260, 560, 2056, 3930, 0, 1490, 2186, 1233, 1947, 1112, 9, 2447, 708, 3917, 6, 23, 3130, 3784, 6724, 112, 5441, 2892, 107, 586, 4948, 1149, 736, 31, 6817, 651, 2120, 9, 7732, 4544, 8, 1824, 1302, 1405, 7208, 1180, 3670, 8, 2, 377, 3, 2585, 0, 71, 108, 4226, 7099, 3019, 3967, 4191] payload_rows_per_dst=[16160, 16104, 16254, 15724, 15744, 13192, 15838, 14856] dup_rows_per_dst=[19782, 26220, 25592, 16234, 14306, 9232, 14228, 12678] combine_rows_per_src=[16160, 16104, 16254, 15724, 15744, 13192, 15838, 14856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130944 payload_rows_remote=114560 dup_rows=131200 dup_rows_remote=114820 dedup_ratio=0.500488 router_tokens_per_expert=[4, 1, 0, 0, 8189, 7, 0, 4, 0, 3, 0, 8180, 176, 0, 0, 1367, 2, 0, 0, 0, 1, 0, 4, 6, 3, 15, 8125, 8186, 8188, 8186, 0, 8188, 1, 3, 5, 0, 7464, 3637, 209, 2799, 8190, 2, 0, 0, 1, 1, 4, 27, 0, 0, 0, 1, 8188, 0, 8191, 2, 8189, 8186, 7929, 1039, 1, 1, 1, 8166] payload_rows_per_dst=[16384, 16384, 16384, 16270, 16378, 16384, 16384, 16376] dup_rows_per_dst=[16398, 16228, 16390, 8462, 23000, 16444, 16380, 17898] combine_rows_per_src=[16384, 16384, 16384, 16270, 16378, 16384, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=881131520 dispatch_meta_bytes=892696 combine_payload_bytes=881131520 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=882024216 ingress_bytes=1057292288 link_bytes=1057292288 token_bytes=1763155736 weight_movement_bytes=176160768 total_nvlink_bytes=1939316504 weight_to_token_ratio=0.099912 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=27648 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.920395477e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130966 payload_rows_remote=114582 dup_rows=131178 dup_rows_remote=112868 dedup_ratio=0.500404 router_tokens_per_expert=[0, 0, 0, 0, 8188, 1, 6, 14, 0, 0, 1, 8176, 283, 0, 0, 701, 0, 0, 0, 2, 6, 0, 0, 4, 2, 10, 8072, 8157, 8192, 8191, 0, 8182, 0, 0, 4, 0, 7890, 5712, 83, 1602, 8182, 1, 1, 0, 1, 0, 0, 4, 0, 1, 0, 2, 8192, 0, 8180, 0, 8192, 8190, 8031, 427, 0, 5, 0, 8184] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16332, 16384, 16384, 16330] dup_rows_per_dst=[16414, 18310, 16398, 18152, 14250, 16378, 16366, 14910] combine_rows_per_src=[16384, 16384, 16384, 16384, 16332, 16384, 16384, 16330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130902 payload_rows_remote=114518 dup_rows=131242 dup_rows_remote=114838 dedup_ratio=0.500648 router_tokens_per_expert=[0, 0, 2, 0, 8190, 2, 4, 0, 1, 0, 0, 8175, 161, 0, 1, 1228, 0, 1, 0, 0, 2, 0, 5, 4, 0, 0, 8172, 8185, 8189, 8187, 0, 8187, 2, 2, 46, 2, 7526, 3580, 2, 2074, 8191, 1, 0, 0, 0, 2, 0, 13, 4, 0, 1, 0, 8189, 1, 8190, 0, 8187, 8191, 8124, 1877, 1, 1, 1, 8168] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16224, 16384, 16384, 16382] dup_rows_per_dst=[16394, 19126, 16388, 11798, 14784, 16404, 16386, 19962] combine_rows_per_src=[16384, 16384, 16384, 16376, 16224, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=882376704 dispatch_meta_bytes=904808 combine_payload_bytes=882376704 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=883281512 ingress_bytes=1146617856 link_bytes=1146617856 token_bytes=1765658216 weight_movement_bytes=264241152 total_nvlink_bytes=2029899368 weight_to_token_ratio=0.149656 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.020833 gemm_active_groups=21 gemm_peak_group_rows=29440 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.420581372e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114658 dup_rows=131102 dup_rows_remote=114704 dedup_ratio=0.500114 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 8189, 185, 0, 0, 1057, 3, 0, 0, 1, 1, 0, 0, 4, 1, 0, 8116, 8164, 8192, 8192, 0, 8190, 0, 0, 9, 0, 8054, 4200, 3, 2624, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 8186, 0, 8192, 8188, 7938, 615, 0, 1, 0, 8176] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16368, 16384, 16384, 16372] dup_rows_per_dst=[16400, 18862, 16398, 16180, 13412, 16384, 16378, 17088] combine_rows_per_src=[16384, 16384, 16384, 16382, 16368, 16384, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130970 payload_rows_remote=114586 dup_rows=131174 dup_rows_remote=110936 dedup_ratio=0.500389 router_tokens_per_expert=[0, 0, 0, 0, 8192, 2, 21, 4, 0, 3, 0, 8182, 214, 0, 0, 1073, 1, 0, 0, 0, 0, 0, 0, 8, 0, 1, 7913, 8188, 8192, 8192, 0, 8178, 0, 0, 3, 0, 7573, 4832, 8, 1225, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8186, 0, 8191, 8191, 7933, 1994, 0, 0, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16282, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16178, 2766, 16338, 15760, 27274, 16250, 16370, 20238] combine_rows_per_src=[16384, 16384, 16384, 16282, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114660 dup_rows=131120 dup_rows_remote=115014 dedup_ratio=0.500183 router_tokens_per_expert=[0, 5, 0, 2, 8188, 12, 16, 10, 1, 0, 0, 8142, 409, 0, 1, 383, 0, 2, 1, 2, 0, 4, 0, 4, 0, 30, 8028, 8177, 8186, 8170, 0, 8187, 0, 10, 4, 4, 7978, 4151, 183, 3372, 8190, 2, 0, 10, 2, 1, 0, 2, 0, 0, 10, 0, 8186, 7, 8183, 0, 8183, 8181, 7934, 355, 1, 2, 2, 8159] payload_rows_per_dst=[16384, 16380, 16384, 16364, 16382, 16384, 16384, 16362] dup_rows_per_dst=[16444, 17830, 16390, 16106, 15022, 16396, 16388, 16544] combine_rows_per_src=[16384, 16380, 16384, 16364, 16382, 16384, 16384, 16362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938475520 dispatch_meta_bytes=917520 combine_payload_bytes=938475520 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939393040 ingress_bytes=1114636288 link_bytes=1114636288 token_bytes=1877868560 weight_movement_bytes=176160768 total_nvlink_bytes=2054029328 weight_to_token_ratio=0.093809 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.241497941e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=10 unused_prefetch_slots=22 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=5 remote_demand_peak=3 slot_utilization=0.312500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 10, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938655744 dispatch_meta_bytes=909800 combine_payload_bytes=938655744 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939565544 ingress_bytes=1114816512 link_bytes=1114816512 token_bytes=1878221288 weight_movement_bytes=176160768 total_nvlink_bytes=2054382056 weight_to_token_ratio=0.093791 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.242507120e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 5, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938131456 dispatch_meta_bytes=917424 combine_payload_bytes=938131456 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939048880 ingress_bytes=1114292224 link_bytes=1114292224 token_bytes=1877180336 weight_movement_bytes=176160768 total_nvlink_bytes=2053341104 weight_to_token_ratio=0.093843 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239571326e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=917448 combine_payload_bytes=939278336 prefetch_bytes=440401920 reduce_bytes=0 egress_bytes=940195784 ingress_bytes=1379680256 link_bytes=1379680256 token_bytes=1879474120 weight_movement_bytes=440401920 total_nvlink_bytes=2319876040 weight_to_token_ratio=0.234322 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725633527e-05 prefetch_bytes_per_gemm_flop=2.466066919e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938688512 dispatch_meta_bytes=902088 combine_payload_bytes=938688512 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=939590600 ingress_bytes=1643331584 link_bytes=1643331584 token_bytes=1878279112 weight_movement_bytes=704643072 total_nvlink_bytes=2582922184 weight_to_token_ratio=0.375154 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201970910e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=918696 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=792723456 egress_bytes=940213416 ingress_bytes=1908178944 link_bytes=1908178944 token_bytes=1879508136 weight_movement_bytes=968884224 total_nvlink_bytes=2848392360 weight_to_token_ratio=0.515499 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.068500557e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131000 payload_rows_remote=114688 dup_rows=131144 dup_rows_remote=118748 dedup_ratio=0.500275 router_tokens_per_expert=[0, 0, 0, 0, 8192, 3, 3, 15, 0, 3, 0, 8189, 355, 0, 0, 1127, 1, 0, 0, 0, 0, 0, 0, 1, 0, 16, 8175, 8188, 8192, 8192, 0, 8187, 0, 0, 19, 0, 7847, 3522, 2, 2964, 8192, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 14, 8191, 0, 8192, 0, 8190, 8192, 8075, 636, 0, 0, 0, 8178] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16312, 16384, 16384, 16384] dup_rows_per_dst=[16426, 19346, 16380, 16374, 12396, 16418, 16410, 17394] combine_rows_per_src=[16384, 16384, 16384, 16384, 16312, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130972 payload_rows_remote=114588 dup_rows=131172 dup_rows_remote=114754 dedup_ratio=0.500381 router_tokens_per_expert=[0, 0, 0, 5, 8187, 4, 16, 0, 9, 5, 0, 8189, 289, 2, 0, 827, 0, 0, 0, 5, 0, 0, 0, 1, 0, 14, 8046, 8182, 8191, 8175, 0, 8185, 0, 0, 34, 0, 7965, 3242, 17, 2555, 8187, 0, 0, 4, 0, 0, 0, 11, 0, 4, 0, 0, 8186, 1, 8176, 0, 8189, 8189, 8128, 1671, 0, 4, 0, 8177] payload_rows_per_dst=[16384, 16384, 16382, 16382, 16340, 16384, 16384, 16332] dup_rows_per_dst=[16418, 18608, 16392, 28488, 11286, 16398, 16394, 7188] combine_rows_per_src=[16384, 16384, 16382, 16382, 16340, 16384, 16384, 16332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114680 dedup_ratio=0.500053 router_tokens_per_expert=[0, 1, 0, 0, 1, 3, 8190, 0, 0, 2, 381, 0, 8191, 0, 1, 8191, 4, 0, 8189, 0, 0, 1, 7836, 0, 8183, 3, 8189, 7646, 0, 0, 1, 1, 1, 83, 2, 4, 8191, 0, 8188, 0, 0, 8191, 8187, 3, 0, 3, 0, 8189, 0, 8188, 1, 1, 1, 13, 1, 8190, 0, 1, 451, 8167, 1, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16384, 16380, 16384, 16380] dup_rows_per_dst=[16388, 17148, 15676, 15290, 16554, 16384, 16406, 17240] combine_rows_per_src=[16384, 16384, 16384, 16378, 16384, 16380, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113288 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 699, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 7347, 0, 8187, 7, 8191, 7717, 0, 0, 0, 0, 0, 11, 1, 5, 8191, 1, 8192, 0, 0, 8192, 8184, 0, 0, 6, 0, 8191, 0, 8192, 0, 0, 0, 1, 0, 8192, 0, 0, 659, 8136, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 17784, 14694, 15438, 16418, 16378, 16386, 17590] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=933744 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940457840 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879981936 weight_movement_bytes=176160768 total_nvlink_bytes=2056142704 weight_to_token_ratio=0.093703 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114710 dedup_ratio=0.500015 router_tokens_per_expert=[1, 0, 0, 0, 1, 0, 8190, 1, 0, 0, 744, 0, 8189, 1, 1, 8188, 1, 1, 8191, 1, 1, 0, 7913, 2, 8173, 1, 8186, 7413, 1, 1, 0, 1, 0, 15, 0, 0, 8190, 5, 8191, 1, 1, 8188, 8187, 0, 1, 1, 0, 8184, 0, 8191, 2, 1, 0, 3, 1, 8191, 0, 1, 340, 8175, 0, 0, 1, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16380, 17862, 15836, 14796, 16420, 16366, 16394, 17022] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114862 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 118, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8106, 0, 8191, 1, 8190, 7648, 0, 0, 0, 0, 0, 143, 1, 0, 8192, 1, 8192, 0, 0, 8192, 8189, 0, 0, 14, 0, 8191, 0, 8192, 0, 0, 0, 4, 0, 8192, 0, 0, 397, 8149, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16620, 16212, 15298, 16674, 16404, 16392, 17090] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127572 payload_rows_remote=111188 dup_rows=134572 dup_rows_remote=117632 dedup_ratio=0.513351 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 8192, 0, 0, 0, 177, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 7965, 0, 8191, 0, 8192, 7936, 0, 0, 0, 0, 0, 20, 0, 0, 8192, 0, 8192, 0, 0, 8192, 8192, 0, 0, 3, 0, 8192, 0, 8192, 0, 0, 0, 3, 0, 8192, 0, 0, 359, 8111, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12884, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16376, 9110, 23468, 19460, 16424, 16376, 16418, 16940] combine_rows_per_src=[16384, 12884, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938704896 dispatch_meta_bytes=917368 combine_payload_bytes=938704896 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939622264 ingress_bytes=1114865664 link_bytes=1114865664 token_bytes=1878327160 weight_movement_bytes=176160768 total_nvlink_bytes=2054487928 weight_to_token_ratio=0.093786 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.242782351e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917416 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940326824 ingress_bytes=1291730944 link_bytes=1291730944 token_bytes=1879736232 weight_movement_bytes=352321536 total_nvlink_bytes=2232057768 weight_to_token_ratio=0.187431 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233154091e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=911904 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940436000 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879960096 weight_movement_bytes=528482304 total_nvlink_bytes=2408442400 weight_to_token_ratio=0.281114 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114672 dup_rows=131102 dup_rows_remote=116882 dedup_ratio=0.500114 router_tokens_per_expert=[4, 0, 4, 0, 0, 0, 8187, 0, 5, 0, 1523, 0, 8188, 3, 3, 8179, 0, 1, 8167, 3, 2, 1, 7034, 0, 8178, 0, 8177, 7110, 5, 2, 0, 0, 0, 3, 11, 0, 8184, 14, 8189, 2, 3, 8187, 8186, 0, 2, 8, 0, 8185, 4, 8189, 2, 4, 3, 1, 3, 8188, 0, 5, 735, 8176, 0, 2, 9, 1] payload_rows_per_dst=[16384, 16384, 16370, 16370, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16380, 19418, 14046, 14220, 16422, 16386, 16404, 17826] combine_rows_per_src=[16384, 16384, 16370, 16370, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917584 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940425296 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879933008 weight_movement_bytes=352321536 total_nvlink_bytes=2232254544 weight_to_token_ratio=0.187412 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=918192 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425904 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879933616 weight_movement_bytes=176160768 total_nvlink_bytes=2056094384 weight_to_token_ratio=0.093706 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=910852096 dispatch_meta_bytes=915280 combine_payload_bytes=910852096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=911767376 ingress_bytes=1087012864 link_bytes=1087012864 token_bytes=1822619472 weight_movement_bytes=176160768 total_nvlink_bytes=1998780240 weight_to_token_ratio=0.096653 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.086818297e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=926216 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940319240 ingress_bytes=1291714560 link_bytes=1291714560 token_bytes=1879712264 weight_movement_bytes=352321536 total_nvlink_bytes=2232033800 weight_to_token_ratio=0.187434 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233062348e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114650 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 0, 4, 0, 0, 8192, 0, 0, 0, 881, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 7860, 0, 8185, 0, 8191, 7360, 0, 0, 0, 0, 0, 24, 0, 0, 8191, 0, 8192, 0, 0, 8192, 8192, 0, 0, 16, 0, 8192, 0, 8192, 0, 0, 0, 10, 0, 8192, 0, 0, 274, 8156, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16392, 18146, 15720, 14714, 16430, 16416, 16404, 16858] combine_rows_per_src=[16384, 16384, 16384, 16376, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=115340 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 1, 8192, 0, 35, 8190, 0, 279, 8, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 8189, 4, 0, 0, 8188, 8187, 8190, 3, 0, 0, 8167, 0, 8192, 0, 2, 0, 8192, 8192, 0, 0, 8139, 7921, 8186, 8190, 0, 30, 8186, 23, 0, 0, 1, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16444, 16948, 16356, 16384, 16338, 15736, 16482] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114656 dup_rows=131104 dup_rows_remote=114714 dedup_ratio=0.500122 router_tokens_per_expert=[0, 0, 0, 1, 0, 2, 8192, 0, 0, 0, 546, 0, 8192, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 7716, 1, 8136, 8, 8191, 7765, 0, 0, 0, 0, 0, 86, 0, 6, 8192, 3, 8192, 0, 0, 8192, 8184, 0, 0, 7, 0, 8186, 0, 8187, 0, 0, 0, 30, 0, 8192, 0, 0, 311, 8170, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16352] dup_rows_per_dst=[16390, 18484, 15434, 21886, 15730, 16370, 16268, 10542] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114604 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 45, 8185, 0, 42, 6, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 8192, 5, 0, 0, 8192, 8182, 8189, 0, 0, 0, 8153, 0, 8192, 0, 0, 0, 8192, 8192, 3, 0, 8179, 8142, 8188, 8192, 0, 38, 8188, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16404, 16468, 16466, 16330, 16368, 16306, 16264, 16466] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[4, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114694 dedup_ratio=0.500008 router_tokens_per_expert=[1, 1, 0, 2, 0, 1, 1, 3, 8, 2, 0, 1, 1, 8188, 0, 0, 8187, 0, 3, 3, 1, 1, 0, 1, 0, 1, 1, 8180, 0, 1, 1, 8190, 0, 1, 1, 8190, 8190, 8191, 1, 0, 0, 8191, 1, 8190, 0, 0, 0, 8190, 8191, 0, 1, 8187, 8186, 8190, 8191, 0, 1, 8190, 8, 1, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16398, 16396, 16388, 16364, 16384, 16380, 16366, 16398] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917320 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940375880 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879834440 weight_movement_bytes=352321536 total_nvlink_bytes=2232155976 weight_to_token_ratio=0.187422 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114582 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 45, 8185, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8191, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8151, 8131, 8192, 8192, 0, 14, 8192, 10, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 16478, 16490, 16344, 16382, 16380, 16180, 16432] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114652 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 41, 8183, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8191, 0, 0, 0, 8186, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8163, 8191, 8191, 8192, 0, 4, 8192, 14, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16278, 15812, 14210, 16350, 16518, 15686, 19798, 16420] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=920096 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940411424 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=1879902752 weight_movement_bytes=704643072 total_nvlink_bytes=2584545824 weight_to_token_ratio=0.374830 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917168 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441264 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965360 weight_movement_bytes=176160768 total_nvlink_bytes=2056126128 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114714 dedup_ratio=0.500038 router_tokens_per_expert=[4, 7, 0, 4, 0, 5, 7, 9, 1, 0, 0, 2, 0, 8173, 0, 58, 8183, 0, 677, 3, 7, 5, 4, 11, 0, 1, 0, 8179, 1, 5, 2, 8188, 0, 6, 6, 8187, 8188, 8180, 0, 1, 8, 8185, 4, 8184, 0, 0, 0, 8185, 8187, 0, 6, 8186, 7127, 8187, 8185, 1, 292, 8183, 40, 3, 0, 0, 0, 5] payload_rows_per_dst=[16384, 16378, 16384, 16384, 16384, 16382, 16384, 16382] dup_rows_per_dst=[16432, 16464, 17770, 16368, 16378, 16380, 14256, 17034] combine_rows_per_src=[16384, 16378, 16384, 16384, 16384, 16382, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=917480 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940179432 ingress_bytes=1115422720 link_bytes=1115422720 token_bytes=1879441384 weight_movement_bytes=176160768 total_nvlink_bytes=2055602152 weight_to_token_ratio=0.093730 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245901632e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917520 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940425232 ingress_bytes=1467990016 link_bytes=1467990016 token_bytes=1879932944 weight_movement_bytes=528482304 total_nvlink_bytes=2408415248 weight_to_token_ratio=0.281118 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220131320e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917080 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441176 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965272 weight_movement_bytes=176160768 total_nvlink_bytes=2056126040 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917360 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441456 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965552 weight_movement_bytes=176160768 total_nvlink_bytes=2056126320 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917568 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940359744 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879801920 weight_movement_bytes=176160768 total_nvlink_bytes=2055962688 weight_to_token_ratio=0.093712 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114706 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 82, 8189, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 8183, 0, 0, 0, 8184, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8123, 8192, 8186, 8192, 0, 36, 8192, 8, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16538, 16400, 16298, 16366, 16368, 16246, 16472] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130996 payload_rows_remote=114612 dup_rows=131148 dup_rows_remote=105704 dedup_ratio=0.500290 router_tokens_per_expert=[0, 8192, 1, 0, 1, 0, 0, 0, 8185, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 8192, 8116, 1, 4, 7, 1046, 0, 0, 144, 8192, 10, 0, 8189, 8192, 1, 0, 8192, 8188, 8192, 0, 0, 0, 0, 0, 0, 152, 0, 0, 7186, 0, 8092, 0, 0, 0, 8188, 8184, 0, 0, 4, 0] payload_rows_per_dst=[16384, 16384, 16384, 16308, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16370, 16058, 7456, 16688, 16376, 25444, 16368] combine_rows_per_src=[16384, 16384, 16384, 16308, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114700 dedup_ratio=0.500008 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 8192, 8161, 0, 0, 4, 1170, 0, 0, 616, 8192, 19, 0, 8192, 8192, 0, 0, 8192, 8188, 8191, 0, 0, 0, 0, 0, 0, 33, 0, 0, 6563, 0, 8165, 0, 2, 0, 8190, 8167, 0, 0, 5, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16384, 16374, 16134, 18670, 17654, 16374, 13140, 16344] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114670 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 57, 8185, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 8185, 6, 0, 0, 8192, 8180, 8192, 0, 0, 0, 8183, 0, 8192, 0, 0, 0, 8192, 8192, 6, 0, 8117, 8183, 8180, 8192, 0, 78, 8183, 8, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16402, 16478, 16400, 16666, 16292, 17236, 16228, 15370] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114802 dedup_ratio=0.500038 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 8192, 8154, 0, 0, 0, 1577, 0, 0, 389, 8192, 6, 0, 8192, 8192, 0, 0, 8192, 8192, 8190, 0, 0, 0, 0, 0, 0, 28, 0, 0, 6364, 0, 8112, 0, 0, 0, 8192, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16374, 16384] dup_rows_per_dst=[16384, 16384, 16280, 19462, 17174, 16380, 12634, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917576 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940441672 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965768 weight_movement_bytes=528482304 total_nvlink_bytes=2408448072 weight_to_token_ratio=0.281113 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114644 dup_rows=131116 dup_rows_remote=114758 dedup_ratio=0.500168 router_tokens_per_expert=[2, 8191, 0, 0, 0, 0, 1, 1, 8189, 0, 8191, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 8085, 1, 8190, 8149, 0, 1, 0, 1980, 0, 0, 322, 8191, 2, 0, 8191, 8191, 0, 1, 8188, 8191, 8178, 1, 0, 1, 0, 0, 0, 54, 1, 0, 6111, 0, 8079, 1, 0, 1, 8190, 8188, 0, 1, 1, 1] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16342, 16384] dup_rows_per_dst=[16382, 16382, 16176, 18950, 17030, 16358, 13458, 16380] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16342, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114642 dedup_ratio=0.500084 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 8192, 8179, 0, 0, 0, 3290, 0, 0, 270, 8192, 5, 0, 8192, 8192, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4845, 0, 8037, 0, 0, 0, 8192, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16362, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16384, 16272, 6576, 16798, 16456, 25776, 16452] combine_rows_per_src=[16384, 16384, 16384, 16362, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114680 dup_rows=131088 dup_rows_remote=113608 dedup_ratio=0.500061 router_tokens_per_expert=[6, 8185, 3, 0, 0, 0, 6, 5, 8188, 0, 8184, 6, 6, 5, 0, 1, 0, 0, 6, 0, 0, 4, 8058, 4, 8186, 8177, 0, 1, 10, 554, 0, 0, 536, 8185, 1, 0, 8186, 8186, 2, 3, 8179, 8184, 8177, 6, 0, 5, 7, 0, 4, 2, 4, 0, 7237, 0, 8179, 5, 0, 7, 8186, 8183, 0, 6, 0, 7] payload_rows_per_dst=[16384, 16384, 16382, 16376, 16378, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16396, 16134, 17480, 17448, 16374, 14478, 16394] combine_rows_per_src=[16384, 16384, 16382, 16376, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917544 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425256 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932968 weight_movement_bytes=176160768 total_nvlink_bytes=2056093736 weight_to_token_ratio=0.093706 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938901504 dispatch_meta_bytes=881264 combine_payload_bytes=938901504 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939782768 ingress_bytes=1115062272 link_bytes=1115062272 token_bytes=1878684272 weight_movement_bytes=176160768 total_nvlink_bytes=2054845040 weight_to_token_ratio=0.093768 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243883273e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917432 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441528 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965624 weight_movement_bytes=352321536 total_nvlink_bytes=2232287160 weight_to_token_ratio=0.187408 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917920 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940360096 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879802272 weight_movement_bytes=352321536 total_nvlink_bytes=2232123808 weight_to_token_ratio=0.187425 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=917608 combine_payload_bytes=939163648 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940081256 ingress_bytes=1291485184 link_bytes=1291485184 token_bytes=1879244904 weight_movement_bytes=352321536 total_nvlink_bytes=2231566440 weight_to_token_ratio=0.187480 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231777938e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917232 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940261104 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879604976 weight_movement_bytes=176160768 total_nvlink_bytes=2055765744 weight_to_token_ratio=0.093722 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=913152 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940371712 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879830272 weight_movement_bytes=352321536 total_nvlink_bytes=2232151808 weight_to_token_ratio=0.187422 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113810 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 8192, 8077, 0, 0, 0, 1266, 0, 0, 439, 8192, 0, 0, 8192, 8192, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 32, 0, 0, 6636, 0, 8186, 0, 0, 0, 8192, 8191, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16266, 18686, 17262, 16384, 13324, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=111908 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 0, 0, 8192, 8187, 0, 0, 0, 8179, 0, 6827, 0, 8192, 0, 15, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 8192, 1, 8192, 0, 0, 0, 8192, 1, 0, 8192, 8192, 0, 0, 0, 8158, 0, 0, 8192, 0, 0, 0, 0, 9, 4, 8191, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 19168, 13632, 16396, 16386, 16386, 16316, 16408] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114754 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 8192, 8190, 0, 0, 0, 8169, 0, 7464, 0, 8192, 0, 17, 0, 0, 0, 0, 8179, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 8187, 0, 0, 0, 8159, 0, 0, 8192, 0, 0, 0, 0, 5, 5, 8192, 0, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 15882, 16928, 16392, 16384, 16374, 16318, 16408] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129822 payload_rows_remote=113438 dup_rows=132322 dup_rows_remote=115936 dedup_ratio=0.504768 router_tokens_per_expert=[0, 8192, 0, 0, 1, 0, 0, 0, 8185, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7931, 0, 8189, 8159, 0, 0, 2, 1413, 0, 0, 634, 8192, 19, 5, 8192, 8192, 1, 0, 8192, 8186, 8191, 0, 0, 0, 0, 0, 0, 22, 0, 0, 6436, 0, 8170, 0, 0, 0, 8192, 8178, 0, 0, 6, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 15134, 16384, 16384, 16384] dup_rows_per_dst=[16386, 16398, 17754, 23900, 12274, 16370, 12872, 16368] combine_rows_per_src=[16384, 16384, 16384, 16384, 15134, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=12 export_slots=12 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 4, 2, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114710 dedup_ratio=0.500084 router_tokens_per_expert=[0, 1, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 1, 797, 0, 0, 8191, 8189, 1, 1, 0, 8162, 0, 7403, 1, 8191, 0, 21, 1, 0, 1, 1, 8189, 8191, 0, 0, 1, 0, 8190, 0, 8191, 1, 1, 0, 8191, 0, 1, 8191, 8190, 1, 1, 0, 8186, 0, 0, 8191, 1, 4, 1, 1, 10, 1, 8184, 0, 0] payload_rows_per_dst=[16384, 16384, 16362, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 17972, 14774, 16424, 16380, 16384, 16376, 16400] combine_rows_per_src=[16384, 16384, 16362, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114688 dup_rows=131076 dup_rows_remote=117622 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 0, 8192, 8192, 0, 0, 0, 8173, 0, 6744, 0, 8192, 0, 9, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 8192, 1, 8192, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 8164, 0, 0, 8192, 0, 0, 0, 0, 10, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 19350, 13454, 16388, 16386, 16384, 16328, 16402] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114662 dedup_ratio=0.500038 router_tokens_per_expert=[0, 2, 8184, 5, 0, 5, 0, 0, 2, 0, 0, 7, 203, 3, 0, 8182, 8174, 8, 1, 0, 8164, 7, 7996, 7, 8185, 4, 20, 2, 5, 4, 0, 8182, 8185, 7, 3, 5, 0, 8182, 1, 8183, 7, 4, 4, 8184, 0, 9, 8185, 8184, 9, 3, 0, 8183, 0, 1, 8184, 7, 0, 7, 9, 9, 0, 8165, 0, 0] payload_rows_per_dst=[16380, 16378, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 16764, 15982, 16420, 16378, 16400, 16390, 16366] combine_rows_per_src=[16380, 16378, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 0, 0, 8190, 8192, 0, 0, 0, 8173, 0, 7125, 0, 8192, 0, 3, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 8189, 0, 0, 0, 8192, 0, 0, 8192, 0, 1, 0, 0, 1, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 6138, 26410, 16294, 16380, 16390, 16690, 16388] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=913992 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=1056964608 egress_bytes=940438088 ingress_bytes=1996488704 link_bytes=1996488704 token_bytes=1879962184 weight_movement_bytes=1056964608 total_nvlink_bytes=2936926792 weight_to_token_ratio=0.562227 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117950337e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=906368 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940397696 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879889024 weight_movement_bytes=176160768 total_nvlink_bytes=2056049792 weight_to_token_ratio=0.093708 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917768 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441864 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965960 weight_movement_bytes=176160768 total_nvlink_bytes=2056126728 weight_to_token_ratio=0.093704 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=929284096 dispatch_meta_bytes=917496 combine_payload_bytes=929284096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=930201592 ingress_bytes=1105444864 link_bytes=1105444864 token_bytes=1859485688 weight_movement_bytes=176160768 total_nvlink_bytes=2035646456 weight_to_token_ratio=0.094736 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.190029803e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917504 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940261376 ingress_bytes=1467826176 link_bytes=1467826176 token_bytes=1879605248 weight_movement_bytes=528482304 total_nvlink_bytes=2408087552 weight_to_token_ratio=0.281167 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219213885e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=929240 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940453336 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879977432 weight_movement_bytes=528482304 total_nvlink_bytes=2408459736 weight_to_token_ratio=0.281111 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441584 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965680 weight_movement_bytes=176160768 total_nvlink_bytes=2056126448 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917360 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940359536 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879801712 weight_movement_bytes=352321536 total_nvlink_bytes=2232123248 weight_to_token_ratio=0.187425 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 0, 0, 8192, 8192, 0, 0, 0, 8162, 0, 7530, 0, 8192, 0, 1, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 8159, 0, 0, 0, 8192, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 17830, 15000, 16384, 16384, 16318, 16384, 16388] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114744 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 5, 8181, 0, 8178, 8179, 0, 0, 0, 0, 0, 8156, 1, 0, 2, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 8190, 8186, 0, 8128, 285, 4, 8192, 0, 0, 7, 0, 8190, 5, 8185, 0, 0, 8192, 8192, 74, 7942, 0, 0, 8192, 0, 0, 2, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16374, 16330, 16346, 16384, 16378, 16848, 16526, 15888] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114386 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 8176, 0, 8181, 8176, 0, 0, 0, 2, 0, 8099, 1, 0, 12, 0, 0, 16, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 0, 0, 8191, 8183, 0, 8092, 446, 4, 8191, 0, 0, 0, 0, 8192, 45, 8188, 0, 0, 8192, 8188, 114, 7796, 0, 0, 8192, 0, 2, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16358, 16334, 16256, 16384, 16376, 17082, 16686, 15596] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114682 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 0, 0, 8192, 8187, 0, 0, 0, 8171, 0, 7462, 0, 8192, 0, 16, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 7, 0, 8192, 8176, 0, 0, 0, 8135, 0, 0, 8192, 0, 1, 0, 0, 9, 5, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16390, 17974, 14922, 16374, 16394, 16436, 16270, 16312] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113500 dedup_ratio=0.500000 router_tokens_per_expert=[1, 0, 2, 1, 0, 1, 1, 8184, 0, 8189, 8090, 0, 0, 1, 1, 1, 8176, 0, 1, 2, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 1, 1, 1, 1, 1, 8191, 8188, 0, 8138, 645, 2, 8191, 0, 0, 1, 1, 8188, 10, 8190, 1, 0, 8190, 8190, 33, 7672, 1, 1, 8191, 0, 0, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16378, 16180, 16374, 16380, 16384, 17572, 16458, 15346] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114752 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8191, 0, 8187, 8178, 0, 0, 0, 0, 0, 8155, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8153, 701, 0, 8192, 0, 0, 0, 0, 8192, 37, 8192, 0, 0, 8192, 8192, 15, 7530, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 16346, 16320, 16384, 16384, 17708, 16488, 15060] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[3, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=14 export_slots=14 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 4, 2, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=103610 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8189, 0, 8190, 8151, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8136, 691, 0, 8192, 0, 0, 0, 0, 8192, 4, 8192, 0, 0, 8192, 8192, 52, 7557, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16360, 16298, 16364, 16384, 16384, 5324, 16496, 27462] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114708 dedup_ratio=0.500046 router_tokens_per_expert=[9, 0, 11, 7, 2, 9, 0, 8185, 0, 8185, 8162, 7, 7, 0, 0, 8, 8169, 1, 7, 0, 0, 0, 1, 0, 1, 1, 0, 5, 0, 8183, 8, 0, 5, 4, 0, 7, 7, 8170, 8185, 0, 8068, 1163, 2, 8183, 6, 4, 17, 8, 8180, 2, 8184, 6, 0, 8183, 8182, 156, 7006, 7, 6, 8181, 3, 0, 0, 8179] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16382, 16378] dup_rows_per_dst=[16424, 16354, 16342, 16376, 16372, 18518, 16674, 14024] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16382, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940441600 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965696 weight_movement_bytes=440401920 total_nvlink_bytes=2320367616 weight_to_token_ratio=0.234261 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917720 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425432 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879933144 weight_movement_bytes=176160768 total_nvlink_bytes=2056093912 weight_to_token_ratio=0.093706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.020833 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916296 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=1233125376 egress_bytes=940440392 ingress_bytes=2172649472 link_bytes=2172649472 token_bytes=1879964488 weight_movement_bytes=1233125376 total_nvlink_bytes=3113089864 weight_to_token_ratio=0.655930 gemm_rows_defined=131456 gemm_rows_charged=135168 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027462 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.216593013e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917480 combine_payload_bytes=939524096 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940441576 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965672 weight_movement_bytes=264241152 total_nvlink_bytes=2144206824 weight_to_token_ratio=0.140556 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=912752 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940436848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879960944 weight_movement_bytes=352321536 total_nvlink_bytes=2232282480 weight_to_token_ratio=0.187409 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917760 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441856 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965952 weight_movement_bytes=352321536 total_nvlink_bytes=2232287488 weight_to_token_ratio=0.187408 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=873192 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940397288 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879921384 weight_movement_bytes=352321536 total_nvlink_bytes=2232242920 weight_to_token_ratio=0.187413 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917536 combine_payload_bytes=939425792 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940343328 ingress_bytes=1203666944 link_bytes=1203666944 token_bytes=1879769120 weight_movement_bytes=264241152 total_nvlink_bytes=2144010272 weight_to_token_ratio=0.140571 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740032451e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8188, 0, 8187, 8185, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8087, 1656, 0, 8192, 0, 0, 0, 0, 8192, 4, 8192, 0, 0, 8192, 8192, 23, 6628, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16376, 16360, 16388, 16384, 16384, 19486, 16438, 13256] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114698 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 1, 0, 1, 8192, 0, 8187, 0, 0, 0, 8192, 0, 0, 8192, 0, 8185, 0, 0, 8191, 0, 8192, 21, 8192, 0, 8192, 0, 59, 8192, 0, 0, 8171, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16374, 16368, 16544, 16342, 16384, 16274, 16398] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114788 dedup_ratio=0.500053 router_tokens_per_expert=[0, 0, 0, 8192, 4, 0, 4, 8192, 0, 8187, 0, 0, 0, 8192, 0, 0, 8192, 1, 8168, 0, 0, 8188, 0, 8190, 16, 8192, 0, 8192, 0, 83, 8187, 0, 0, 8142, 0, 8186, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8142, 0, 0, 0, 0, 26, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[16400, 16374, 16336, 16574, 16286, 16382, 16298, 16436] combine_rows_per_src=[16384, 16384, 16376, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130600 payload_rows_remote=114216 dup_rows=131544 dup_rows_remote=115124 dedup_ratio=0.501801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 6, 8183, 0, 8178, 8187, 0, 0, 0, 1, 0, 8154, 0, 0, 7, 0, 0, 17, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 9, 0, 0, 8192, 8185, 0, 7963, 1191, 6, 8192, 0, 0, 0, 0, 8190, 20, 8188, 0, 0, 8192, 8186, 26, 7223, 0, 0, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 15912, 16384] dup_rows_per_dst=[16420, 16870, 16686, 16396, 16410, 18534, 15782, 14446] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 15912, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114714 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 8179, 0, 0, 8191, 0, 8192, 12, 8192, 0, 8192, 0, 105, 8192, 0, 0, 8165, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16358, 16618, 16328, 16382, 16234, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114690 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8191, 2, 1, 1, 8191, 1, 8190, 1, 1, 0, 8191, 1, 0, 8191, 1, 8174, 1, 1, 8191, 1, 8189, 2, 8191, 0, 8191, 1, 41, 8190, 0, 10, 8169, 0, 8167, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 8184, 1, 0, 1, 0, 20, 8191, 1, 0, 0, 8187, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16386, 16352, 16466, 16310, 16382, 16374, 16414] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130590 payload_rows_remote=114206 dup_rows=131554 dup_rows_remote=115112 dedup_ratio=0.501839 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 8192, 0, 8192, 0, 8192, 0, 168, 8192, 0, 0, 8165, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16372, 16384, 16384, 15914, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16350, 16382, 16380, 15986, 17006, 16388, 16620, 16442] combine_rows_per_src=[16372, 16384, 16384, 15914, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114684 dedup_ratio=0.500038 router_tokens_per_expert=[9, 0, 4, 8184, 1, 7, 0, 8183, 8, 8185, 8, 8, 0, 8184, 8, 0, 8183, 0, 8187, 6, 5, 8183, 0, 8180, 0, 8182, 4, 8183, 9, 9, 8185, 4, 3, 8181, 0, 8109, 9, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 6, 33, 2, 10, 0, 8179, 8, 0, 5, 0, 21, 8184, 2, 0, 23, 8175, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16380] dup_rows_per_dst=[16392, 16418, 16372, 16398, 16226, 16386, 16460, 16430] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 5, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=10 unused_prefetch_slots=22 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=5 remote_demand_peak=3 slot_utilization=0.312500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 10, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917544 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=940441640 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965736 weight_movement_bytes=264241152 total_nvlink_bytes=2144206888 weight_to_token_ratio=0.140556 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917848 combine_payload_bytes=939409408 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940327256 ingress_bytes=1291730944 link_bytes=1291730944 token_bytes=1879736664 weight_movement_bytes=352321536 total_nvlink_bytes=2232058200 weight_to_token_ratio=0.187431 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233154091e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=935657472 dispatch_meta_bytes=917360 combine_payload_bytes=935657472 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=936574832 ingress_bytes=1199898624 link_bytes=1199898624 token_bytes=1872232304 weight_movement_bytes=264241152 total_nvlink_bytes=2136473456 weight_to_token_ratio=0.141137 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.718931432e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 4, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917608 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940441704 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879965800 weight_movement_bytes=616562688 total_nvlink_bytes=2496528488 weight_to_token_ratio=0.327965 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917512 combine_payload_bytes=939524096 prefetch_bytes=440401920 reduce_bytes=0 egress_bytes=940441608 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965704 weight_movement_bytes=440401920 total_nvlink_bytes=2320367624 weight_to_token_ratio=0.234261 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=2.466066919e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=935575552 dispatch_meta_bytes=917272 combine_payload_bytes=935575552 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=936492824 ingress_bytes=1287897088 link_bytes=1287897088 token_bytes=1872068376 weight_movement_bytes=352321536 total_nvlink_bytes=2224389912 weight_to_token_ratio=0.188199 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.211686098e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917448 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=940359624 ingress_bytes=1732165632 link_bytes=1732165632 token_bytes=1879801800 weight_movement_bytes=792723456 total_nvlink_bytes=2672525256 weight_to_token_ratio=0.421706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699404498e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114668 dup_rows=131092 dup_rows_remote=114698 dedup_ratio=0.500076 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 11, 8192, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 8187, 8175, 0, 0, 0, 8182, 8192, 0, 0, 1, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 14, 0, 1, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 8192, 8191, 0, 4, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16364, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16378, 16394, 16400, 16350, 16386, 16408, 16386, 16390] combine_rows_per_src=[16384, 16384, 16384, 16364, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114688 dup_rows=131076 dup_rows_remote=114790 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 8180, 0, 0, 8192, 0, 8192, 1, 8192, 0, 8192, 0, 51, 8192, 0, 0, 8143, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16360, 16488, 16286, 16384, 16406, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=114716 dedup_ratio=0.500092 router_tokens_per_expert=[5, 0, 0, 7, 0, 0, 4, 0, 8188, 0, 11, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8185, 8172, 0, 0, 0, 8180, 8192, 0, 0, 3, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 8186, 0, 10, 0, 0, 0, 8192, 8190, 0, 0, 0, 8191, 0, 0, 8192, 8192, 0, 7, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16360, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16414, 16398, 16382, 16344, 16390, 16392, 16380, 16396] combine_rows_per_src=[16384, 16384, 16384, 16360, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114674 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 1, 0, 6, 8192, 0, 8185, 0, 0, 0, 8192, 0, 0, 8192, 0, 8157, 0, 0, 8192, 0, 8192, 7, 8192, 0, 8192, 0, 35, 8192, 0, 0, 8152, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8192, 0, 0, 0, 0, 22, 8192, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16398, 16370, 16554, 16468, 16304, 16384, 16406, 16188] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114702 dedup_ratio=0.500038 router_tokens_per_expert=[1, 0, 0, 3, 1, 0, 1, 3, 8190, 0, 7, 8191, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 8190, 8180, 1, 1, 0, 8187, 8191, 1, 1, 0, 1, 8189, 8191, 0, 1, 1, 8191, 0, 0, 0, 8189, 0, 1, 1, 4, 1, 8191, 8190, 1, 0, 0, 8190, 7, 1, 8191, 8184, 0, 1, 0, 8190] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16398, 16396, 16386, 16364, 16384, 16380, 16390, 16384] combine_rows_per_src=[16384, 16384, 16384, 16376, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114700 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8184, 0, 0, 0, 8182, 8192, 0, 0, 1, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 9, 0, 2, 0, 8192, 8192, 0, 1, 0, 8192, 0, 0, 8192, 8191, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16372, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16396, 16384, 16360, 16386, 16402, 16390, 16382] combine_rows_per_src=[16384, 16384, 16384, 16372, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114584 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8181, 0, 0, 0, 8190, 8192, 0, 0, 2, 0, 8192, 8192, 1, 0, 0, 8192, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 8192, 8192, 0, 1, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16318, 16302, 16322, 16382, 16356, 16354, 16550, 16492] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114674 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 1, 6, 0, 0, 16, 8171, 0, 0, 8180, 0, 8, 10, 0, 0, 9, 0, 10, 0, 10, 2, 5, 8184, 8182, 9, 8, 0, 8181, 8182, 13, 2, 8, 1, 8179, 8178, 0, 7, 9, 8184, 0, 10, 0, 8184, 0, 0, 5, 0, 9, 8174, 8184, 8, 0, 2, 8183, 103, 8, 8182, 8083, 0, 0, 0, 8182] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16394, 16358, 16416, 16402, 16384, 16382, 16368, 16372] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=917464 combine_payload_bytes=939360256 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940277720 ingress_bytes=1203601408 link_bytes=1203601408 token_bytes=1879637976 weight_movement_bytes=264241152 total_nvlink_bytes=2143879128 weight_to_token_ratio=0.140581 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739665477e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=10 unused_prefetch_slots=22 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=5 remote_demand_peak=3 slot_utilization=0.312500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 6, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917912 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940442008 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879966104 weight_movement_bytes=176160768 total_nvlink_bytes=2056126872 weight_to_token_ratio=0.093704 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.023674 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=917520 combine_payload_bytes=939327488 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940245008 ingress_bytes=1467809792 link_bytes=1467809792 token_bytes=1879572496 weight_movement_bytes=528482304 total_nvlink_bytes=2408054800 weight_to_token_ratio=0.281172 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219122141e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[2, 0, 3, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917448 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940441544 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965640 weight_movement_bytes=352321536 total_nvlink_bytes=2232287176 weight_to_token_ratio=0.187408 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917520 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940359696 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879801872 weight_movement_bytes=352321536 total_nvlink_bytes=2232123408 weight_to_token_ratio=0.187425 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917504 combine_payload_bytes=939425792 prefetch_bytes=440401920 reduce_bytes=0 egress_bytes=940343296 ingress_bytes=1379827712 link_bytes=1379827712 token_bytes=1879769088 weight_movement_bytes=440401920 total_nvlink_bytes=2320171008 weight_to_token_ratio=0.234285 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726459219e-05 prefetch_bytes_per_gemm_flop=2.466066919e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917072 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940408400 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879899728 weight_movement_bytes=528482304 total_nvlink_bytes=2408382032 weight_to_token_ratio=0.281123 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917432 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940408760 ingress_bytes=1556054016 link_bytes=1556054016 token_bytes=1879900088 weight_movement_bytes=616562688 total_nvlink_bytes=2496462776 weight_to_token_ratio=0.327976 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713252961e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114704 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 8176, 8192, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8176, 8189, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 10, 0, 0, 0, 0, 8166, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 8190, 0, 14, 0, 3, 0, 0, 8192, 8192, 8192, 1, 0, 21] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16352, 16368, 16384, 16430, 16332, 16368, 16410, 16428] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114682 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8167, 0, 0, 0, 8188, 8192, 0, 0, 7, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 16, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16392, 16388, 16384, 16334, 16398, 16416, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16376, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114644 dedup_ratio=0.500084 router_tokens_per_expert=[8192, 8192, 8171, 8192, 0, 0, 0, 0, 0, 22, 0, 0, 0, 8185, 8183, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 9, 0, 0, 0, 0, 8127, 0, 0, 8192, 0, 8187, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 1, 8188, 0, 27, 0, 5, 0, 0, 8192, 8192, 8192, 3, 0, 23] payload_rows_per_dst=[16384, 16372, 16384, 16384, 16384, 16374, 16384, 16384] dup_rows_per_dst=[16342, 16410, 16384, 16446, 16254, 16372, 16450, 16436] combine_rows_per_src=[16384, 16372, 16384, 16384, 16384, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114688 dedup_ratio=0.500038 router_tokens_per_expert=[6, 0, 0, 0, 0, 0, 6, 0, 8186, 0, 16, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8185, 8175, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 8192, 8192, 2, 0, 0, 8192, 0, 0, 0, 8187, 0, 2, 0, 0, 0, 8192, 8192, 0, 0, 3, 8187, 0, 0, 8192, 8192, 0, 11, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16374, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16394, 16444, 16398, 16622, 16388, 16338, 16226, 16272] combine_rows_per_src=[16384, 16384, 16384, 16374, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=114692 dedup_ratio=0.500008 router_tokens_per_expert=[8191, 8190, 8188, 8190, 0, 0, 0, 1, 0, 17, 1, 1, 0, 8189, 8190, 8187, 8191, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 8, 2, 0, 0, 1, 1, 8170, 1, 0, 8191, 0, 8190, 0, 1, 0, 0, 1, 0, 8190, 1, 1, 0, 8190, 0, 5, 0, 1, 0, 1, 8191, 8185, 8191, 1, 0, 4] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16374, 16404, 16390, 16406, 16344, 16382, 16394, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 8186, 8192, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8183, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 1, 0, 0, 8147, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 21, 0, 2, 0, 0, 8192, 8192, 8192, 0, 0, 9] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16372, 16406, 16384, 16398, 16296, 16384, 16430, 16402] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114402 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 8191, 8192, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8188, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8188, 0, 0, 8192, 0, 8192, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16254, 16104, 16334, 16392, 16650, 16402, 16266, 16670] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917568 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441664 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965760 weight_movement_bytes=528482304 total_nvlink_bytes=2408448064 weight_to_token_ratio=0.281113 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114660 dup_rows=131102 dup_rows_remote=114716 dedup_ratio=0.500114 router_tokens_per_expert=[8182, 8172, 8184, 8178, 12, 11, 5, 7, 0, 107, 0, 8, 0, 8183, 8185, 8089, 8180, 3, 9, 10, 11, 1, 7, 3, 0, 8, 9, 0, 7, 0, 0, 0, 0, 5, 0, 8177, 9, 0, 8180, 0, 8183, 0, 7, 0, 0, 9, 0, 8179, 0, 12, 0, 8181, 4, 7, 37, 0, 0, 1, 8176, 8144, 8182, 0, 0, 8] payload_rows_per_dst=[16384, 16382, 16384, 16382, 16384, 16384, 16382, 16360] dup_rows_per_dst=[16398, 16392, 16428, 16386, 16358, 16372, 16470, 16298] combine_rows_per_src=[16384, 16382, 16384, 16382, 16384, 16384, 16382, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917448 combine_payload_bytes=939458560 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940376008 ingress_bytes=1203699712 link_bytes=1203699712 token_bytes=1879834568 weight_movement_bytes=264241152 total_nvlink_bytes=2144075720 weight_to_token_ratio=0.140566 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740215938e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917240 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940261112 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879604984 weight_movement_bytes=176160768 total_nvlink_bytes=2055765752 weight_to_token_ratio=0.093722 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917464 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940359640 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879801816 weight_movement_bytes=352321536 total_nvlink_bytes=2232123352 weight_to_token_ratio=0.187425 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441616 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965712 weight_movement_bytes=176160768 total_nvlink_bytes=2056126480 weight_to_token_ratio=0.093704 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916360 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940440456 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879964552 weight_movement_bytes=528482304 total_nvlink_bytes=2408446856 weight_to_token_ratio=0.281113 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=917504 combine_payload_bytes=939294720 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940212224 ingress_bytes=1291616256 link_bytes=1291616256 token_bytes=1879506944 weight_movement_bytes=352321536 total_nvlink_bytes=2231828480 weight_to_token_ratio=0.187454 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232511886e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 8177, 1281, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 8192, 16, 0, 9, 0, 0, 8192, 8170, 8191, 0, 6752, 0, 0, 0, 0, 8192, 1, 0, 0, 171, 0, 8192, 8192, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[18946, 16380, 16384, 16438, 13460, 16728, 16382, 16354] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114804 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 35, 0, 0, 0, 8177, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 8133, 0, 0, 8192, 0, 8192, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 24, 0, 1, 0, 0, 8192, 8192, 8192, 0, 0, 8] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16424, 16384, 16392, 16268, 16386, 16434, 16400] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114478 dedup_ratio=0.500008 router_tokens_per_expert=[8192, 8192, 8179, 8192, 0, 0, 0, 0, 0, 26, 0, 0, 0, 8174, 8180, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 12, 0, 1, 0, 0, 8132, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 8191, 0, 13, 0, 6, 0, 0, 8192, 8192, 8192, 1, 0, 29] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16596, 16578, 16384, 16224, 16570, 16420, 16220, 16082] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114710 dedup_ratio=0.500084 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 8177, 2390, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 8192, 13, 0, 7, 0, 0, 8182, 8189, 8192, 0, 5669, 0, 0, 0, 0, 8192, 0, 0, 0, 139, 0, 8192, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191] payload_rows_per_dst=[16364, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[4780, 16378, 16384, 16416, 27716, 16662, 16384, 16374] combine_rows_per_src=[16364, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114584 dedup_ratio=0.500000 router_tokens_per_expert=[8184, 8173, 2, 1, 1, 0, 8184, 5035, 8191, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 3, 0, 8191, 2, 1, 8190, 13, 1, 134, 0, 7, 8187, 8189, 8185, 0, 3001, 0, 0, 1, 1, 8191, 0, 6, 0, 49, 0, 8189, 8191, 8191, 0, 1, 1, 8176, 1, 0, 0, 1, 1, 0, 1, 1, 0, 8190] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15494, 16386, 16390, 16650, 16922, 16488, 16356, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114604 dedup_ratio=0.500061 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 8192, 4042, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 311, 0, 0, 8192, 8188, 8192, 0, 3811, 0, 0, 0, 0, 8192, 0, 0, 0, 32, 0, 8192, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16368, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8000, 16380, 16320, 16368, 24640, 16328, 16568, 16484] combine_rows_per_src=[16368, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940441616 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965712 weight_movement_bytes=264241152 total_nvlink_bytes=2144206864 weight_to_token_ratio=0.140556 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 8191, 3630, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 13, 0, 0, 8192, 8177, 8192, 0, 4452, 0, 0, 0, 0, 8192, 0, 0, 0, 116, 0, 8192, 8192, 8192, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[23642, 16382, 16384, 16412, 8874, 16616, 16378, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114686 dup_rows=131080 dup_rows_remote=114468 dedup_ratio=0.500031 router_tokens_per_expert=[8032, 8082, 4, 9, 6, 0, 8179, 3696, 8182, 10, 2, 0, 0, 0, 8, 9, 0, 0, 0, 10, 7, 31, 0, 8183, 0, 4, 8173, 39, 0, 126, 0, 71, 8182, 8182, 8181, 0, 4430, 0, 0, 10, 10, 8181, 0, 37, 0, 112, 0, 8180, 8176, 8182, 0, 12, 5, 8116, 7, 0, 0, 10, 11, 0, 12, 10, 4, 8159] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16384, 16384, 16384, 16382] dup_rows_per_dst=[21070, 16400, 16442, 16612, 11264, 16656, 16248, 16388] combine_rows_per_src=[16384, 16384, 16384, 16378, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 4, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917968 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940442064 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879966160 weight_movement_bytes=176160768 total_nvlink_bytes=2056126928 weight_to_token_ratio=0.093704 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=916656 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=940424368 ingress_bytes=1644150784 link_bytes=1644150784 token_bytes=1879932080 weight_movement_bytes=704643072 total_nvlink_bytes=2584575152 weight_to_token_ratio=0.374824 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206558088e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917504 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940261376 ingress_bytes=1291665408 link_bytes=1291665408 token_bytes=1879605248 weight_movement_bytes=352321536 total_nvlink_bytes=2231926784 weight_to_token_ratio=0.187444 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232787117e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917088 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940441184 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965280 weight_movement_bytes=352321536 total_nvlink_bytes=2232286816 weight_to_token_ratio=0.187409 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917104 combine_payload_bytes=939393024 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940310128 ingress_bytes=1203634176 link_bytes=1203634176 token_bytes=1879703152 weight_movement_bytes=264241152 total_nvlink_bytes=2143944304 weight_to_token_ratio=0.140576 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739848964e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=131456 gemm_rows_charged=135168 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027462 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=916616 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940424328 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932040 weight_movement_bytes=176160768 total_nvlink_bytes=2056092808 weight_to_token_ratio=0.093706 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=114704 dedup_ratio=0.500008 router_tokens_per_expert=[0, 9, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8185, 8191, 0, 8192, 28, 0, 0, 0, 0, 8183, 0, 8192, 1, 0, 0, 0, 8192, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 8190, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 7, 0, 8163, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16402, 16370, 16422, 16386, 16380, 16390, 16384, 16340] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113434 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 8191, 3365, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 1, 0, 0, 8192, 8150, 8192, 0, 4765, 0, 0, 0, 0, 8192, 0, 0, 0, 103, 0, 8192, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14922, 16382, 16384, 16390, 17638, 16590, 16384, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=106826 dedup_ratio=0.500031 router_tokens_per_expert=[8192, 8192, 0, 0, 0, 0, 8164, 3350, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 8192, 22, 0, 0, 0, 0, 8191, 8167, 8192, 0, 4836, 0, 0, 0, 0, 8192, 0, 0, 0, 46, 0, 8192, 8192, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16380] dup_rows_per_dst=[24254, 16384, 16384, 15792, 10562, 15004, 16372, 16328] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114710 dedup_ratio=0.500053 router_tokens_per_expert=[0, 15, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 8183, 8183, 0, 8192, 21, 0, 0, 0, 0, 8192, 0, 8192, 13, 0, 0, 0, 8192, 8192, 8187, 0, 4, 0, 0, 0, 1, 0, 0, 0, 8192, 2, 8187, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 8189, 0, 0, 0, 8192, 0, 7, 0, 8161, 0] payload_rows_per_dst=[16384, 16370, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16416, 16362, 16426, 16410, 16382, 16378, 16376, 16336] combine_rows_per_src=[16384, 16370, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114368 dedup_ratio=0.500023 router_tokens_per_expert=[0, 1, 0, 2, 1, 0, 1, 0, 0, 8191, 0, 1, 0, 0, 8189, 8190, 2, 8189, 8, 0, 0, 0, 0, 8023, 1, 8191, 1, 0, 1, 2, 8190, 8191, 8189, 1, 0, 1, 1, 3, 1, 0, 1, 167, 8189, 0, 8190, 0, 0, 0, 8191, 1, 8189, 1, 0, 1, 8190, 1, 0, 1, 8191, 2, 2, 0, 8183, 1] payload_rows_per_dst=[16384, 16382, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16390, 16378, 16064, 16388, 16388, 16710, 16384, 16376] combine_rows_per_src=[16384, 16382, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917568 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940441664 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879965760 weight_movement_bytes=616562688 total_nvlink_bytes=2496528448 weight_to_token_ratio=0.327965 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 3, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114492 dedup_ratio=0.500061 router_tokens_per_expert=[0, 5, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 0, 0, 8192, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1, 0, 8182, 0] payload_rows_per_dst=[16384, 16368, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16228, 16298, 16326, 16550, 16306, 16402, 16382, 16596] combine_rows_per_src=[16384, 16368, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[0, 16, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 8192, 0, 8192, 17, 0, 0, 0, 0, 8173, 0, 8192, 0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1, 0, 8173, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16416, 16378, 16380, 16384, 16384, 16398, 16384, 16348] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114702 dedup_ratio=0.500053 router_tokens_per_expert=[0, 0, 0, 15, 10, 0, 7, 3, 0, 8181, 0, 9, 0, 0, 8182, 8184, 13, 8178, 0, 0, 0, 0, 7, 7702, 11, 8180, 0, 2, 3, 12, 8175, 8181, 8174, 11, 0, 24, 5, 24, 0, 0, 2, 476, 8148, 0, 8181, 0, 6, 0, 8183, 4, 8175, 13, 6, 7, 8182, 9, 0, 4, 8182, 10, 0, 0, 8185, 6] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16384, 16378, 16384, 16384] dup_rows_per_dst=[16408, 16366, 15424, 16384, 16456, 17248, 16410, 16390] combine_rows_per_src=[16384, 16384, 16376, 16384, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=912488 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940436584 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879960680 weight_movement_bytes=616562688 total_nvlink_bytes=2496523368 weight_to_token_ratio=0.327966 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=886024 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940344584 ingress_bytes=1556021248 link_bytes=1556021248 token_bytes=1879803144 weight_movement_bytes=616562688 total_nvlink_bytes=2496365832 weight_to_token_ratio=0.327993 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713069474e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917536 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940326944 ingress_bytes=1467891712 link_bytes=1467891712 token_bytes=1879736352 weight_movement_bytes=528482304 total_nvlink_bytes=2408218656 weight_to_token_ratio=0.281147 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219580859e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=916200 combine_payload_bytes=939474944 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940391144 ingress_bytes=1203716096 link_bytes=1203716096 token_bytes=1879866088 weight_movement_bytes=264241152 total_nvlink_bytes=2144107240 weight_to_token_ratio=0.140564 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.023674 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740307682e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=916656 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940309680 ingress_bytes=1115553792 link_bytes=1115553792 token_bytes=1879702704 weight_movement_bytes=176160768 total_nvlink_bytes=2055863472 weight_to_token_ratio=0.093717 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246635580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=940441616 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965712 weight_movement_bytes=264241152 total_nvlink_bytes=2144206864 weight_to_token_ratio=0.140556 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.025568 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917504 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940326912 ingress_bytes=1467891712 link_bytes=1467891712 token_bytes=1879736320 weight_movement_bytes=528482304 total_nvlink_bytes=2408218624 weight_to_token_ratio=0.281147 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219580859e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=102414 dedup_ratio=0.500000 router_tokens_per_expert=[0, 2, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 25, 8192, 0, 0, 0, 8192, 0, 0, 4, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 390, 0, 0, 85, 0, 8192, 8192, 0, 1587, 0, 0, 0, 0, 8192, 8192, 8192, 0, 8192, 8171] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16368, 28658, 16384, 16434, 16392, 17150, 3344, 16342] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[0, 9, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 8192, 18, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 8192, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 8192, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1, 0, 8171, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16402, 16384, 16412, 16384, 16380, 16386, 16380, 16344] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=868408 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940392504 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879916600 weight_movement_bytes=176160768 total_nvlink_bytes=2056077368 weight_to_token_ratio=0.093707 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114662 dedup_ratio=0.500000 router_tokens_per_expert=[0, 11, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 8178, 8187, 0, 8192, 34, 0, 0, 0, 0, 8179, 0, 8192, 11, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8187, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 14, 0, 8157, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16410, 16570, 17630, 16186, 16396, 15166, 16372, 16342] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114030 dedup_ratio=0.500038 router_tokens_per_expert=[1, 4, 0, 8188, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1777, 1, 8188, 3, 0, 1, 8191, 1, 1, 8190, 2, 0, 1, 14, 8189, 0, 0, 1, 8191, 2, 0, 0, 8189, 0, 1, 0, 1, 3, 6, 8190, 0, 1, 330, 0, 1, 5, 0, 8189, 8141, 2, 6127, 0, 0, 1, 0, 8191, 8190, 8180, 2, 8187, 8186] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16376, 16382] dup_rows_per_dst=[16388, 4044, 16394, 16408, 16384, 17052, 28058, 16354] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16376, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=104552 dedup_ratio=0.500023 router_tokens_per_expert=[0, 4, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2756, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 16, 8192, 1, 0, 0, 8192, 1, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 381, 0, 0, 33, 0, 8192, 8192, 0, 5038, 0, 0, 0, 0, 8192, 8188, 8192, 0, 8192, 8178] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16378] dup_rows_per_dst=[16366, 5512, 16384, 16418, 16390, 17128, 26526, 16354] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114688 dup_rows=131080 dup_rows_remote=108984 dedup_ratio=0.500031 router_tokens_per_expert=[0, 35, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 10, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 787, 0, 0, 11, 0, 8192, 8192, 0, 5753, 0, 0, 0, 0, 8192, 8187, 8192, 0, 8192, 8187] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16376] dup_rows_per_dst=[16396, 3212, 16294, 16034, 16316, 12820, 27912, 22096] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114514 dedup_ratio=0.500061 router_tokens_per_expert=[9, 3, 2, 8185, 5, 0, 11, 0, 0, 0, 11, 0, 0, 5769, 9, 8142, 27, 0, 6, 8181, 10, 11, 8175, 14, 26, 12, 93, 8161, 0, 0, 7, 8180, 50, 0, 0, 8162, 0, 3, 0, 16, 19, 34, 8167, 0, 11, 39, 0, 8, 27, 0, 8149, 7801, 21, 2717, 5, 0, 12, 0, 8180, 8171, 8090, 14, 8154, 8173] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16374, 16384] dup_rows_per_dst=[16388, 27776, 16464, 16574, 16444, 16480, 4768, 16194] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16374, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 6, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114698 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3360, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 10, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 247, 0, 0, 36, 0, 8192, 8192, 0, 4560, 0, 0, 0, 0, 8192, 8185, 8192, 0, 8192, 8183] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] dup_rows_per_dst=[16384, 23104, 16384, 16404, 16384, 16868, 9192, 16362] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940441616 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965712 weight_movement_bytes=264241152 total_nvlink_bytes=2144206864 weight_to_token_ratio=0.140556 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917400 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441496 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965592 weight_movement_bytes=352321536 total_nvlink_bytes=2232287128 weight_to_token_ratio=0.187409 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=914832 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940357008 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879799184 weight_movement_bytes=176160768 total_nvlink_bytes=2055959952 weight_to_token_ratio=0.093713 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.025568 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=876936 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940351880 ingress_bytes=1291796480 link_bytes=1291796480 token_bytes=1879826824 weight_movement_bytes=352321536 total_nvlink_bytes=2232148360 weight_to_token_ratio=0.187422 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233521066e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=894688 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940418784 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879942880 weight_movement_bytes=880803840 total_nvlink_bytes=2760746720 weight_to_token_ratio=0.468527 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=916744 combine_payload_bytes=939393024 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940309768 ingress_bytes=1203634176 link_bytes=1203634176 token_bytes=1879702792 weight_movement_bytes=264241152 total_nvlink_bytes=2143943944 weight_to_token_ratio=0.140576 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739848964e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917504 combine_payload_bytes=939442176 prefetch_bytes=0 reduce_bytes=88080384 egress_bytes=940359680 ingress_bytes=1027522560 link_bytes=1027522560 token_bytes=1879801856 weight_movement_bytes=88080384 total_nvlink_bytes=1967882240 weight_to_token_ratio=0.046856 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.753697427e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114718 dedup_ratio=0.500000 router_tokens_per_expert=[15, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 8192, 0, 8184, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 7, 0, 1, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8187, 8191, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16414, 16354, 16368, 16400, 16386, 16372, 16384, 16394] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917624 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441720 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965816 weight_movement_bytes=352321536 total_nvlink_bytes=2232287352 weight_to_token_ratio=0.187408 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 143, 0, 0, 74, 0, 8192, 8192, 0, 6259, 0, 0, 0, 0, 8192, 8192, 8192, 0, 8192, 8174] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16390, 17968, 16384, 16386, 16384, 16670, 14542, 16348] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114680 dedup_ratio=0.500000 router_tokens_per_expert=[0, 13, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 17, 8192, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 273, 0, 0, 83, 0, 8192, 8192, 0, 5641, 0, 0, 0, 0, 8192, 8190, 8192, 0, 8192, 8174] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16392, 20798, 16384, 16414, 16384, 16904, 16748, 11048] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114818 dedup_ratio=0.500023 router_tokens_per_expert=[3, 0, 0, 0, 8190, 8191, 8181, 1, 1, 0, 0, 8188, 2, 0, 1, 1, 8190, 0, 8190, 0, 0, 8187, 8191, 10, 0, 1, 8132, 0, 2, 1, 47, 1, 0, 2, 1, 4, 3, 8188, 1, 1, 8189, 8128, 1, 1, 2, 0, 1, 8191, 0, 8189, 8190, 0, 1, 1, 1, 8188, 1, 75, 0, 0, 0, 1, 1, 0] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16368, 16382, 16392, 16366, 16398, 16260, 16378, 16534] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131036 payload_rows_remote=114652 dup_rows=131108 dup_rows_remote=114724 dedup_ratio=0.500137 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8189, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16348, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16352, 16416, 15032, 16362, 16262, 17852, 16448, 16384] combine_rows_per_src=[16384, 16384, 16348, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130998 payload_rows_remote=114614 dup_rows=131146 dup_rows_remote=114744 dedup_ratio=0.500282 router_tokens_per_expert=[0, 0, 0, 0, 8183, 8178, 8116, 0, 2, 9, 0, 8185, 17, 0, 0, 11, 8179, 0, 8182, 0, 0, 8176, 8181, 65, 0, 11, 7840, 0, 0, 10, 340, 9, 0, 10, 9, 34, 24, 8150, 5, 11, 8164, 7539, 7, 11, 16, 0, 0, 8181, 11, 8178, 8175, 1, 10, 0, 4, 8181, 8, 645, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[16356, 16384, 16364, 16384, 16384, 16360, 16384, 16382] dup_rows_per_dst=[16232, 16420, 16482, 16402, 16464, 15114, 16380, 17652] combine_rows_per_src=[16356, 16384, 16364, 16384, 16384, 16360, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=114706 dedup_ratio=0.500008 router_tokens_per_expert=[14, 0, 0, 0, 8190, 8192, 8192, 1, 0, 0, 0, 8172, 0, 1, 2, 0, 8192, 1, 8183, 1, 0, 8192, 8192, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 10, 0, 0, 0, 0, 8192, 0, 0, 8189, 8192, 0, 0, 0, 0, 1, 8192, 0, 8190, 8192, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16412, 16348, 16370, 16398, 16404, 16380, 16368, 16394] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[2, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16382, 16380, 16386, 16386, 16382, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917472 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441568 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965664 weight_movement_bytes=176160768 total_nvlink_bytes=2056126432 weight_to_token_ratio=0.093704 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=918000 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940392944 ingress_bytes=1379876864 link_bytes=1379876864 token_bytes=1879867888 weight_movement_bytes=440401920 total_nvlink_bytes=2320269808 weight_to_token_ratio=0.234273 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726734449e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=917504 combine_payload_bytes=939229184 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940146688 ingress_bytes=1291550720 link_bytes=1291550720 token_bytes=1879375872 weight_movement_bytes=352321536 total_nvlink_bytes=2231697408 weight_to_token_ratio=0.187467 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232144912e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938917888 dispatch_meta_bytes=917432 combine_payload_bytes=938917888 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939835320 ingress_bytes=1115078656 link_bytes=1115078656 token_bytes=1878753208 weight_movement_bytes=176160768 total_nvlink_bytes=2054913976 weight_to_token_ratio=0.093765 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243975017e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917576 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940441672 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965768 weight_movement_bytes=528482304 total_nvlink_bytes=2408448072 weight_to_token_ratio=0.281113 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=940441616 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965712 weight_movement_bytes=704643072 total_nvlink_bytes=2584608784 weight_to_token_ratio=0.374817 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114716 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 8192, 8178, 72, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8120, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16412, 16356, 16528, 16384, 16384, 16384, 16240, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917616 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441712 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965808 weight_movement_bytes=176160768 total_nvlink_bytes=2056126576 weight_to_token_ratio=0.093704 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16380, 16384, 16388, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114532 dedup_ratio=0.500046 router_tokens_per_expert=[8, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8179, 0, 0, 7, 0, 8192, 0, 8185, 0, 0, 8192, 8192, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16372, 16384] dup_rows_per_dst=[16552, 16386, 17726, 16374, 16384, 16352, 16362, 14948] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16372, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114690 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 8190, 3, 1, 1, 1, 1, 1, 0, 8188, 8185, 31, 0, 1, 1, 0, 8190, 2, 0, 0, 0, 8191, 2, 3, 0, 8190, 1, 8191, 18, 0, 8191, 3, 1, 8191, 8191, 0, 8190, 0, 0, 0, 2, 1, 0, 1, 8191, 8144, 0, 1, 0, 8190, 0, 1, 1, 1, 0, 0, 8187, 8191, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16386, 16372, 16448, 16390, 16424, 16384, 16290, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114688 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 8192, 8192, 62, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8128, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16294, 16384, 15626, 16368, 16394, 16386, 17236, 16386] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=114712 dedup_ratio=0.500099 router_tokens_per_expert=[0, 0, 14, 5, 0, 0, 8177, 0, 11, 12, 11, 11, 5, 0, 8170, 8181, 3, 0, 9, 6, 0, 8183, 0, 0, 1, 0, 8181, 14, 9, 0, 8180, 0, 8182, 1, 0, 8182, 24, 12, 8181, 8176, 6, 8181, 0, 0, 0, 16, 0, 0, 10, 8181, 8165, 0, 10, 0, 8172, 5, 11, 10, 14, 1, 2, 8157, 8179, 11] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16384, 16384, 16364, 16384] dup_rows_per_dst=[16378, 16418, 16380, 16386, 16404, 16386, 16360, 16386] combine_rows_per_src=[16378, 16384, 16384, 16384, 16384, 16384, 16364, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114866 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 18, 0, 0, 0, 0, 0, 0, 8192, 8174, 89, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 3, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8188, 0, 2, 0, 0, 1, 0, 0, 8192, 8103, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16420, 16348, 16558, 16390, 16384, 16382, 16206, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114628 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 8192, 8192, 30, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8158, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16392, 16384, 16444, 16384, 16384, 16384, 16316, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441584 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965680 weight_movement_bytes=176160768 total_nvlink_bytes=2056126448 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=916832 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940342624 ingress_bytes=1379827712 link_bytes=1379827712 token_bytes=1879768416 weight_movement_bytes=440401920 total_nvlink_bytes=2320170336 weight_to_token_ratio=0.234285 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726459219e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940425216 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932928 weight_movement_bytes=264241152 total_nvlink_bytes=2144174080 weight_to_token_ratio=0.140559 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917496 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=0 egress_bytes=940425208 ingress_bytes=1027588096 link_bytes=1027588096 token_bytes=1879932920 weight_movement_bytes=88080384 total_nvlink_bytes=1968013304 weight_to_token_ratio=0.046853 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.754064401e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=917496 combine_payload_bytes=939311104 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940228600 ingress_bytes=1291632640 link_bytes=1291632640 token_bytes=1879539704 weight_movement_bytes=352321536 total_nvlink_bytes=2231861240 weight_to_token_ratio=0.187451 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232603630e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[0, 1146, 0, 7045, 0, 8165, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8175, 0, 0, 8192, 5, 8192, 0, 0, 0, 0, 6, 8187, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16328, 16380, 16384, 16360, 16386, 16438, 16412, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918216 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940442312 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879966408 weight_movement_bytes=352321536 total_nvlink_bytes=2232287944 weight_to_token_ratio=0.187408 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917264 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441360 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965456 weight_movement_bytes=176160768 total_nvlink_bytes=2056126224 weight_to_token_ratio=0.093704 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940441616 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965712 weight_movement_bytes=352321536 total_nvlink_bytes=2232287248 weight_to_token_ratio=0.187408 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 47, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8145, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16478, 16384, 16384, 16384, 16290, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114664 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 8192, 8178, 115, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8189, 0, 0, 0, 0, 4, 0, 0, 8192, 8078, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16412, 16426, 16610, 16388, 16318, 16382, 16156, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114680 dedup_ratio=0.500031 router_tokens_per_expert=[0, 918, 1, 7226, 1, 8183, 0, 0, 8190, 1, 8191, 1, 2, 0, 1, 8181, 4, 0, 8190, 8190, 1, 8190, 0, 1, 3, 0, 8188, 0, 0, 8188, 1, 8191, 0, 1, 1, 91, 3, 8190, 0, 0, 1, 0, 1, 0, 5, 2, 1, 1, 0, 1, 0, 0, 0, 2, 8191, 6, 0, 8185, 1, 8190, 8153, 0, 2, 1] payload_rows_per_dst=[16382, 16384, 16384, 16382, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16276, 16368, 16388, 16378, 16568, 16400, 16398, 16304] combine_rows_per_src=[16382, 16384, 16384, 16382, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113604 dedup_ratio=0.500000 router_tokens_per_expert=[0, 1763, 0, 6432, 0, 8182, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8185, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16370, 16378, 15808, 16944, 15452, 16260, 16392, 17468] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130998 payload_rows_remote=114618 dup_rows=131146 dup_rows_remote=114840 dedup_ratio=0.500282 router_tokens_per_expert=[0, 2512, 9, 5660, 3, 8184, 0, 0, 8182, 9, 8180, 0, 15, 0, 0, 8080, 157, 0, 8179, 8165, 7, 8177, 8, 21, 8, 10, 8182, 0, 0, 8143, 0, 8178, 0, 10, 2, 386, 0, 8181, 0, 0, 12, 0, 11, 0, 5, 0, 14, 7, 2, 13, 0, 0, 0, 15, 8180, 0, 0, 8109, 8, 8177, 7844, 0, 17, 10] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16314] dup_rows_per_dst=[16352, 16184, 16688, 16306, 17128, 16426, 16400, 15662] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 0, 1, 0, 3, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29964973984 prefetch_bytes=2818572288 reduce_bytes=1761607680 weight_movement_bytes=4580179968 egress_bytes=14989768608 ingress_bytes=19555385344 link_bytes=19555385344 total_nvlink_bytes=34545153952 weight_to_token_ratio=0.152851 prefetch_share_of_ingress=0.144133 dedup_ratio=0.501688 prefetch_slots=64 unused_prefetch_slots=448 slot_utilization=0.125000 reduce_slots=20 remote_demand=32 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.023615 gemm_inner_pad_upper_bound=27051 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.843874694e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114644 dedup_ratio=0.500084 router_tokens_per_expert=[0, 624, 0, 7546, 0, 8156, 1, 0, 8190, 0, 8192, 1, 0, 0, 1, 8187, 3, 0, 8189, 8192, 0, 8192, 0, 0, 0, 0, 8167, 0, 0, 8192, 5, 8192, 0, 0, 0, 0, 9, 8188, 0, 0, 0, 2, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 34, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16362, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16292, 16376, 16384, 16344, 16394, 16470, 16450, 16384] combine_rows_per_src=[16362, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 1980, 0, 6213, 0, 8173, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8184, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16348, 16384, 16384, 16368, 16384, 16424, 16396, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=940441600 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879965696 weight_movement_bytes=880803840 total_nvlink_bytes=2760769536 weight_to_token_ratio=0.468521 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917392 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940408720 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879900048 weight_movement_bytes=176160768 total_nvlink_bytes=2056060816 weight_to_token_ratio=0.093708 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917440 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940376000 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879834560 weight_movement_bytes=352321536 total_nvlink_bytes=2232156096 weight_to_token_ratio=0.187422 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=913168 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940437264 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879961360 weight_movement_bytes=528482304 total_nvlink_bytes=2408443664 weight_to_token_ratio=0.281113 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938950656 dispatch_meta_bytes=917832 combine_payload_bytes=938950656 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=939868488 ingress_bytes=1467432960 link_bytes=1467432960 token_bytes=1878819144 weight_movement_bytes=528482304 total_nvlink_bytes=2407301448 weight_to_token_ratio=0.281284 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217012039e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917240 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940261112 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879604984 weight_movement_bytes=176160768 total_nvlink_bytes=2055765752 weight_to_token_ratio=0.093722 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441600 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965696 weight_movement_bytes=528482304 total_nvlink_bytes=2408448000 weight_to_token_ratio=0.281113 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29970942944 prefetch_bytes=2730491904 reduce_bytes=2906652672 weight_movement_bytes=5637144576 egress_bytes=14992788448 ingress_bytes=20615299072 link_bytes=20615299072 total_nvlink_bytes=35608087520 weight_to_token_ratio=0.188087 prefetch_share_of_ingress=0.132450 dedup_ratio=0.501601 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=33 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.023674 gemm_inner_pad_upper_bound=27051 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.214816847e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29873296608 prefetch_bytes=2730491904 reduce_bytes=3170893824 weight_movement_bytes=5901385728 egress_bytes=14943917280 ingress_bytes=20830765056 link_bytes=20830765056 total_nvlink_bytes=35774682336 weight_to_token_ratio=0.197547 prefetch_share_of_ingress=0.131080 dedup_ratio=0.503078 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=36 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.023142 gemm_inner_pad_upper_bound=28448 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.290224320e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29957710416 prefetch_bytes=2730491904 reduce_bytes=4227858432 weight_movement_bytes=6958350336 egress_bytes=14986191440 ingress_bytes=21929869312 link_bytes=21929869312 total_nvlink_bytes=36916060752 weight_to_token_ratio=0.232272 prefetch_share_of_ingress=0.124510 dedup_ratio=0.501930 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=48 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.022786 gemm_inner_pad_upper_bound=28829 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.674882136e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29955157848 prefetch_bytes=2730491904 reduce_bytes=3611295744 weight_movement_bytes=6341787648 egress_bytes=14984867672 ingress_bytes=21312077824 link_bytes=21312077824 total_nvlink_bytes=36296945496 weight_to_token_ratio=0.211709 prefetch_share_of_ingress=0.128119 dedup_ratio=0.501830 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=41 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.022846 gemm_inner_pad_upper_bound=28956 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.458671232e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29998516984 prefetch_bytes=3347054592 reduce_bytes=2025848832 weight_movement_bytes=5372903424 egress_bytes=15006599928 ingress_bytes=20364820480 link_bytes=20364820480 total_nvlink_bytes=35371420408 weight_to_token_ratio=0.179106 prefetch_share_of_ingress=0.164355 dedup_ratio=0.501411 prefetch_slots=76 unused_prefetch_slots=436 slot_utilization=0.148438 reduce_slots=23 remote_demand=38 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.023438 gemm_inner_pad_upper_bound=27940 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.127155875e-05 prefetch_bytes_per_gemm_flop=1.171381787e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 697, 0, 7505, 0, 8177, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 8168, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 22, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16374, 16384, 16384, 16336, 16384, 16398, 16428, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131030 payload_rows_remote=114646 dup_rows=131114 dup_rows_remote=114358 dedup_ratio=0.500160 router_tokens_per_expert=[0, 1149, 0, 7053, 0, 8155, 0, 0, 8191, 0, 8192, 0, 0, 0, 1, 8191, 0, 0, 8187, 8192, 0, 8192, 0, 0, 0, 0, 8156, 0, 0, 8192, 6, 8192, 0, 0, 0, 0, 8, 8186, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 26, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16342, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16756, 16618, 16652, 16084, 16376, 16256, 16196, 16176] combine_rows_per_src=[16384, 16384, 16384, 16342, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939180032 dispatch_meta_bytes=916016 combine_payload_bytes=939180032 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940096048 ingress_bytes=1115340800 link_bytes=1115340800 token_bytes=1879276080 weight_movement_bytes=176160768 total_nvlink_bytes=2055436848 weight_to_token_ratio=0.093739 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245442914e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29962387448 prefetch_bytes=2378170368 reduce_bytes=3435134976 weight_movement_bytes=5813305344 egress_bytes=14988525560 ingress_bytes=20787167232 link_bytes=20787167232 total_nvlink_bytes=35775692792 weight_to_token_ratio=0.194020 prefetch_share_of_ingress=0.114406 dedup_ratio=0.501778 prefetch_slots=54 unused_prefetch_slots=458 slot_utilization=0.105469 reduce_slots=39 remote_demand=27 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.022786 gemm_inner_pad_upper_bound=29083 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.274966219e-05 prefetch_bytes_per_gemm_flop=8.322975852e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29933454528 prefetch_bytes=3347054592 reduce_bytes=1673527296 weight_movement_bytes=5020581888 egress_bytes=14974043328 ingress_bytes=19979993088 link_bytes=19979993088 total_nvlink_bytes=34954036416 weight_to_token_ratio=0.167725 prefetch_share_of_ingress=0.167520 dedup_ratio=0.502177 prefetch_slots=76 unused_prefetch_slots=436 slot_utilization=0.148438 reduce_slots=19 remote_demand=38 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.023201 gemm_inner_pad_upper_bound=27940 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.992476327e-05 prefetch_bytes_per_gemm_flop=1.171381787e-05 anomalies=none - [2026-09-16 21:39:33] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 612.7 | throughput per GPU (TFLOP/s/GPU): 544.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.519648E+00 | loss scale: 1.0 | grad norm: 5.762 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122662 payload_rows_remote=106828 dup_rows=139482 dup_rows_remote=117830 dedup_ratio=0.532082 router_tokens_per_expert=[5139, 311, 403, 140, 3530, 11, 3029, 71, 1222, 6019, 1122, 668, 1845, 1635, 0, 405, 636, 2768, 2769, 1299, 15, 1240, 1954, 2774, 46, 43, 5676, 2194, 6827, 558, 5714, 3720, 41, 2247, 5351, 295, 630, 93, 5785, 878, 2653, 34, 6561, 2815, 8, 3826, 1548, 1441, 6834, 335, 1761, 19, 52, 1174, 7, 1734, 2, 58, 86, 3973, 6830, 2410, 2098, 5710] payload_rows_per_dst=[14540, 14824, 14636, 15266, 15882, 15632, 15834, 16048] dup_rows_per_dst=[10728, 11008, 12274, 15452, 19942, 22140, 21652, 26286] combine_rows_per_src=[14540, 14824, 14636, 15266, 15882, 15632, 15834, 16048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123290 payload_rows_remote=107442 dup_rows=138854 dup_rows_remote=115128 dedup_ratio=0.529686 router_tokens_per_expert=[6076, 209, 344, 114, 3215, 1, 3051, 176, 1455, 6122, 1327, 474, 1775, 2794, 3, 497, 803, 2232, 3246, 1373, 4, 1118, 1948, 2702, 98, 20, 6245, 1983, 6111, 358, 6168, 2954, 49, 1883, 5032, 289, 804, 138, 6523, 657, 3417, 66, 6692, 3132, 5, 4200, 1129, 1427, 6868, 145, 1543, 2, 11, 1159, 2, 1304, 2, 51, 237, 2963, 6781, 1758, 2097, 5710] payload_rows_per_dst=[16202, 15778, 15848, 14744, 15794, 14694, 15540, 14690] dup_rows_per_dst=[23732, 21030, 23726, 11208, 18006, 12058, 18148, 10946] combine_rows_per_src=[16202, 15778, 15848, 14744, 15794, 14694, 15540, 14690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121992 payload_rows_remote=105978 dup_rows=140152 dup_rows_remote=111504 dedup_ratio=0.534637 router_tokens_per_expert=[6020, 177, 565, 183, 3007, 0, 2692, 134, 1414, 6306, 1087, 486, 1724, 2561, 0, 434, 933, 2277, 2658, 1247, 9, 1095, 1677, 2678, 55, 41, 5645, 2209, 5741, 451, 5636, 3277, 104, 2631, 5289, 678, 757, 137, 5924, 991, 3339, 70, 6432, 2749, 12, 4132, 1318, 1122, 6551, 208, 2102, 1, 39, 1458, 0, 2002, 0, 108, 169, 4206, 6850, 1383, 2620, 5271] payload_rows_per_dst=[14558, 15208, 13864, 15154, 16052, 15474, 15668, 16014] dup_rows_per_dst=[10998, 12816, 11284, 13730, 21780, 20360, 20536, 28648] combine_rows_per_src=[14558, 15208, 13864, 15154, 16052, 15474, 15668, 16014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123442 payload_rows_remote=107822 dup_rows=138702 dup_rows_remote=123694 dedup_ratio=0.529106 router_tokens_per_expert=[5657, 222, 714, 178, 3451, 1, 3052, 128, 1113, 6334, 1249, 753, 1991, 2430, 0, 639, 789, 2255, 3429, 1789, 1, 828, 1923, 2652, 31, 35, 5695, 2605, 6279, 423, 5569, 2938, 39, 2579, 5122, 452, 769, 136, 5547, 670, 3589, 52, 6128, 3496, 1, 3751, 1276, 1300, 6947, 178, 1659, 8, 44, 1577, 9, 1498, 2, 115, 56, 3000, 6767, 1645, 2219, 5288] payload_rows_per_dst=[14906, 15420, 14560, 15696, 15620, 15628, 15760, 15852] dup_rows_per_dst=[12004, 13598, 13996, 17672, 15008, 23558, 20638, 22228] combine_rows_per_src=[14906, 15420, 14560, 15696, 15620, 15628, 15760, 15852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123368 payload_rows_remote=107620 dup_rows=138776 dup_rows_remote=116020 dedup_ratio=0.529388 router_tokens_per_expert=[5418, 204, 597, 185, 2911, 3, 2978, 55, 1299, 6407, 1295, 714, 2562, 3166, 1, 585, 701, 2387, 2937, 1664, 2, 1075, 2152, 3175, 47, 11, 5503, 2775, 6679, 561, 6264, 2727, 49, 2408, 4493, 392, 482, 67, 5002, 1434, 2776, 70, 6180, 3426, 3, 4251, 1162, 1384, 6438, 173, 1331, 2, 9, 1559, 0, 1270, 2, 85, 254, 2953, 6895, 2306, 1941, 5235] payload_rows_per_dst=[14608, 15686, 14456, 15984, 15168, 15748, 15760, 15958] dup_rows_per_dst=[10094, 16372, 13730, 19792, 13486, 22756, 19162, 23384] combine_rows_per_src=[14608, 15686, 14456, 15984, 15168, 15748, 15760, 15958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123220 payload_rows_remote=108254 dup_rows=138924 dup_rows_remote=125214 dedup_ratio=0.529953 router_tokens_per_expert=[5944, 174, 384, 269, 3846, 0, 3787, 141, 1570, 6389, 872, 915, 2122, 2228, 0, 435, 1041, 1756, 2610, 1528, 11, 906, 1978, 2446, 63, 27, 5499, 2444, 6438, 531, 6070, 2788, 52, 2526, 4586, 596, 724, 116, 5963, 891, 3018, 192, 6254, 3111, 11, 4947, 1188, 1225, 6748, 149, 1898, 4, 49, 1455, 10, 1319, 2, 90, 225, 3263, 6333, 1234, 2313, 5368] payload_rows_per_dst=[16214, 15350, 15420, 14966, 15874, 15248, 15704, 14444] dup_rows_per_dst=[25096, 13712, 19016, 13710, 18452, 17510, 20436, 10992] combine_rows_per_src=[16214, 15350, 15420, 14966, 15874, 15248, 15704, 14444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123610 payload_rows_remote=107350 dup_rows=138534 dup_rows_remote=113176 dedup_ratio=0.528465 router_tokens_per_expert=[5837, 77, 247, 229, 3613, 1, 3193, 39, 1251, 6758, 1770, 814, 1456, 2440, 0, 498, 515, 2850, 3577, 1442, 2, 1378, 1602, 2921, 24, 58, 5855, 2475, 6280, 437, 5496, 2492, 41, 2519, 5445, 586, 463, 115, 5547, 882, 2714, 17, 6597, 3262, 2, 3696, 1345, 1432, 7107, 303, 1480, 7, 11, 1489, 0, 1431, 0, 69, 152, 3061, 6367, 1891, 2418, 4996] payload_rows_per_dst=[16206, 16260, 15958, 16074, 15642, 13964, 15182, 14324] dup_rows_per_dst=[23000, 25358, 24554, 19884, 15554, 10972, 8582, 10630] combine_rows_per_src=[16206, 16260, 15958, 16074, 15642, 13964, 15182, 14324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=875134976 dispatch_meta_bytes=898632 combine_payload_bytes=875134976 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=876033608 ingress_bytes=1051295744 link_bytes=1051295744 token_bytes=1751168584 weight_movement_bytes=176160768 total_nvlink_bytes=1927329352 weight_to_token_ratio=0.100596 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=27264 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.886817334e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=880164864 dispatch_meta_bytes=890280 combine_payload_bytes=880164864 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=881055144 ingress_bytes=1056325632 link_bytes=1056325632 token_bytes=1761220008 weight_movement_bytes=176160768 total_nvlink_bytes=1937380776 weight_to_token_ratio=0.100022 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=12416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.914982607e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=868171776 dispatch_meta_bytes=869928 combine_payload_bytes=868171776 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=869041704 ingress_bytes=1308573696 link_bytes=1308573696 token_bytes=1737213480 weight_movement_bytes=440401920 total_nvlink_bytes=2177615400 weight_to_token_ratio=0.253511 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=21376 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.327466472e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=883277824 dispatch_meta_bytes=926064 combine_payload_bytes=883277824 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=884203888 ingress_bytes=1059438592 link_bytes=1059438592 token_bytes=1767481712 weight_movement_bytes=176160768 total_nvlink_bytes=1943642480 weight_to_token_ratio=0.099668 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=23296 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.932413884e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=881623040 dispatch_meta_bytes=894560 combine_payload_bytes=881623040 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=882517600 ingress_bytes=1322024960 link_bytes=1322024960 token_bytes=1764140640 weight_movement_bytes=440401920 total_nvlink_bytes=2204542560 weight_to_token_ratio=0.249641 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.022727 gemm_active_groups=18 gemm_peak_group_rows=16000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.402787936e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=886816768 dispatch_meta_bytes=933872 combine_payload_bytes=886816768 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=887750640 ingress_bytes=1591459840 link_bytes=1591459840 token_bytes=1774567408 weight_movement_bytes=704643072 total_nvlink_bytes=2479210480 weight_to_token_ratio=0.397079 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=22528 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.911510796e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122810 payload_rows_remote=106584 dup_rows=139334 dup_rows_remote=116302 dedup_ratio=0.531517 router_tokens_per_expert=[5213, 119, 421, 150, 3326, 0, 3455, 119, 651, 6245, 1183, 987, 2340, 1922, 0, 633, 1052, 2533, 2970, 1294, 5, 837, 1984, 2813, 58, 40, 5688, 2107, 6221, 455, 5713, 3884, 54, 2753, 4883, 482, 394, 83, 5698, 595, 3237, 36, 6101, 3056, 0, 4347, 1598, 1155, 6858, 180, 1539, 1, 13, 1324, 1, 1855, 0, 48, 84, 3575, 6826, 2276, 2110, 5492] payload_rows_per_dst=[16226, 16220, 15880, 16052, 15588, 14550, 14892, 13402] dup_rows_per_dst=[23032, 23504, 22942, 26760, 14296, 11852, 8650, 8298] combine_rows_per_src=[16226, 16220, 15880, 16052, 15588, 14550, 14892, 13402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=879411200 dispatch_meta_bytes=882104 combine_payload_bytes=879411200 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=880293304 ingress_bytes=1055571968 link_bytes=1055571968 token_bytes=1759704504 weight_movement_bytes=176160768 total_nvlink_bytes=1935865272 weight_to_token_ratio=0.100108 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=25472 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.910762403e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=873136128 dispatch_meta_bytes=891544 combine_payload_bytes=873136128 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=874027672 ingress_bytes=1049296896 link_bytes=1049296896 token_bytes=1747163800 weight_movement_bytes=176160768 total_nvlink_bytes=1923324568 weight_to_token_ratio=0.100827 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=23424 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.875624620e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129774 payload_rows_remote=113390 dup_rows=132370 dup_rows_remote=98716 dedup_ratio=0.504951 router_tokens_per_expert=[0, 1, 0, 0, 8192, 16, 8190, 0, 3, 1, 0, 5110, 8185, 0, 0, 19, 1, 0, 0, 0, 0, 0, 2, 7182, 0, 8140, 0, 7308, 169, 8190, 10, 879, 0, 0, 3, 0, 0, 8, 0, 7593, 8182, 1, 0, 0, 0, 0, 0, 4127, 0, 0, 0, 8175, 8192, 0, 1, 0, 8168, 8173, 519, 8167, 0, 0, 0, 8165] payload_rows_per_dst=[16384, 16372, 16384, 15098, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16414, 10264, 14332, 2986, 30132, 8236, 16352, 33654] combine_rows_per_src=[16384, 16372, 16384, 15098, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114666 dup_rows=131110 dup_rows_remote=114908 dedup_ratio=0.500145 router_tokens_per_expert=[15, 6, 3, 0, 8172, 2, 8090, 2, 56, 0, 0, 3789, 8192, 0, 0, 24, 5, 1, 1, 19, 22, 13, 1, 7820, 0, 8150, 0, 7176, 512, 8067, 0, 729, 0, 5, 1, 6, 39, 1, 22, 7241, 8180, 1, 0, 1, 4, 0, 0, 5401, 0, 0, 0, 8096, 8180, 3, 6, 0, 8179, 8176, 418, 8078, 6, 3, 1, 8157] payload_rows_per_dst=[16378, 16384, 16384, 16382, 16380, 16376, 16368, 16382] dup_rows_per_dst=[16202, 7738, 15732, 16752, 14384, 10798, 16202, 33302] combine_rows_per_src=[16378, 16384, 16384, 16382, 16380, 16376, 16368, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114684 dup_rows=131078 dup_rows_remote=117126 dedup_ratio=0.500023 router_tokens_per_expert=[0, 0, 0, 0, 8189, 2, 8153, 1, 8, 2, 0, 5864, 8185, 0, 0, 97, 0, 0, 1, 0, 0, 0, 1, 7054, 0, 8151, 2, 6838, 532, 8187, 1, 567, 0, 0, 0, 0, 2, 1, 1, 6976, 8188, 10, 0, 0, 2, 0, 0, 4467, 1, 4, 3, 8172, 8191, 0, 9, 0, 8186, 8190, 487, 8181, 0, 0, 0, 8166] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16382, 16382, 16384, 16384] dup_rows_per_dst=[16308, 11928, 14108, 15798, 13952, 8952, 16376, 33656] combine_rows_per_src=[16382, 16384, 16384, 16384, 16382, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125636 payload_rows_remote=109252 dup_rows=136508 dup_rows_remote=122982 dedup_ratio=0.520737 router_tokens_per_expert=[0, 0, 2, 0, 8191, 0, 8166, 0, 26, 1, 0, 4877, 8178, 0, 1, 80, 8, 1, 3, 0, 0, 2, 18, 6736, 0, 8051, 2, 4946, 1110, 8190, 1, 1923, 0, 0, 3, 0, 16, 8, 2, 7431, 8186, 2, 0, 0, 0, 3, 0, 5343, 0, 4, 1, 8177, 8189, 0, 5, 0, 8192, 8187, 451, 8185, 1, 3, 1, 8169] payload_rows_per_dst=[16382, 16384, 16384, 16382, 16382, 16384, 16384, 10954] dup_rows_per_dst=[16336, 22982, 13526, 21256, 14918, 26762, 16664, 4064] combine_rows_per_src=[16382, 16384, 16384, 16382, 16382, 16384, 16384, 10954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131026 payload_rows_remote=114642 dup_rows=131118 dup_rows_remote=121780 dedup_ratio=0.500175 router_tokens_per_expert=[1, 0, 0, 0, 8188, 7, 8192, 1, 29, 0, 0, 5095, 8173, 0, 0, 82, 1, 0, 0, 2, 0, 0, 0, 6849, 0, 8138, 0, 6414, 1301, 8179, 2, 1153, 0, 0, 0, 0, 33, 4, 3, 6893, 8191, 19, 0, 0, 0, 0, 0, 4651, 4, 0, 0, 8099, 8192, 0, 42, 2, 8184, 8187, 391, 8191, 0, 0, 0, 8179] payload_rows_per_dst=[16384, 16362, 16384, 16360, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16394, 10396, 13694, 17656, 13840, 9338, 16294, 33506] combine_rows_per_src=[16384, 16362, 16384, 16360, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114754 dedup_ratio=0.500038 router_tokens_per_expert=[0, 1, 2, 0, 8189, 2, 8147, 0, 39, 0, 0, 5217, 8191, 0, 1, 134, 3, 0, 0, 2, 0, 0, 0, 7108, 0, 8165, 4, 6579, 465, 8185, 14, 1129, 0, 0, 5, 0, 25, 0, 0, 7396, 8185, 0, 0, 0, 2, 0, 0, 4201, 2, 0, 0, 8130, 8186, 0, 87, 0, 8186, 8189, 555, 8184, 0, 0, 0, 8162] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16380, 16382, 16384] dup_rows_per_dst=[16298, 12634, 14224, 16328, 14838, 11254, 16428, 29078] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16380, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=115772 payload_rows_remote=99388 dup_rows=146372 dup_rows_remote=119434 dedup_ratio=0.558365 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 8172, 0, 5, 1, 0, 5140, 8180, 0, 0, 155, 3, 0, 0, 0, 0, 0, 3, 7704, 2, 8088, 0, 4889, 549, 8191, 0, 582, 0, 0, 1, 0, 1, 3, 0, 7276, 8192, 2, 0, 0, 0, 0, 0, 6058, 0, 0, 0, 8187, 8191, 0, 57, 0, 8185, 8192, 559, 8189, 0, 0, 0, 8123] payload_rows_per_dst=[16384, 16384, 16384, 16350, 16384, 16384, 16384, 1118] dup_rows_per_dst=[16344, 26938, 16470, 27084, 14560, 28490, 16486, 0] combine_rows_per_src=[16384, 16384, 16384, 16350, 16384, 16384, 16384, 1118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=14 export_slots=14 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 2, 4, 2, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=928890880 dispatch_meta_bytes=848424 combine_payload_bytes=928890880 prefetch_bytes=0 reduce_bytes=1233125376 egress_bytes=929739304 ingress_bytes=2162016256 link_bytes=2162016256 token_bytes=1858630184 weight_movement_bytes=1233125376 total_nvlink_bytes=3091755560 weight_to_token_ratio=0.663459 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=23296 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.210638856e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=918296 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940262168 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879606040 weight_movement_bytes=176160768 total_nvlink_bytes=2055766808 weight_to_token_ratio=0.093722 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=927240 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940418568 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879909896 weight_movement_bytes=352321536 total_nvlink_bytes=2232231432 weight_to_token_ratio=0.187414 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=894992384 dispatch_meta_bytes=928936 combine_payload_bytes=894992384 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=895921320 ingress_bytes=1247313920 link_bytes=1247313920 token_bytes=1790913704 weight_movement_bytes=352321536 total_nvlink_bytes=2143235240 weight_to_token_ratio=0.196727 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017045 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.984437298e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=945688 combine_payload_bytes=939147264 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940092952 ingress_bytes=1115308032 link_bytes=1115308032 token_bytes=1879240216 weight_movement_bytes=176160768 total_nvlink_bytes=2055400984 weight_to_token_ratio=0.093740 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245259427e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=814186496 dispatch_meta_bytes=875288 combine_payload_bytes=814186496 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=815061784 ingress_bytes=990347264 link_bytes=990347264 token_bytes=1629248280 weight_movement_bytes=176160768 total_nvlink_bytes=1805409048 weight_to_token_ratio=0.108124 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.023674 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.545531287e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917728 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940359904 ingress_bytes=1467924480 link_bytes=1467924480 token_bytes=1879802080 weight_movement_bytes=528482304 total_nvlink_bytes=2408284384 weight_to_token_ratio=0.281137 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=29056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219764346e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=115568 payload_rows_remote=99184 dup_rows=146576 dup_rows_remote=129790 dedup_ratio=0.559143 router_tokens_per_expert=[0, 0, 0, 0, 8191, 1, 8179, 0, 19, 1, 0, 5855, 8189, 0, 0, 66, 4, 0, 1, 0, 1, 0, 24, 6590, 1, 8107, 3, 4894, 1954, 8192, 1, 512, 0, 1, 3, 0, 1, 0, 0, 7223, 8185, 2, 1, 1, 1, 0, 0, 5461, 0, 3, 1, 8125, 8190, 0, 17, 0, 8169, 8191, 447, 8188, 0, 0, 0, 8077] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16372, 16384, 894] dup_rows_per_dst=[16786, 28226, 28990, 14562, 14456, 27268, 16288, 0] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16372, 16384, 894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=812515328 dispatch_meta_bytes=915896 combine_payload_bytes=812515328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=813431224 ingress_bytes=988676096 link_bytes=988676096 token_bytes=1625946552 weight_movement_bytes=176160768 total_nvlink_bytes=1802107320 weight_to_token_ratio=0.108344 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.536173444e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114688 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8179, 0, 8192, 0, 0, 8192, 0, 0, 8190, 0, 0, 8122, 3, 0, 4590, 74, 8189, 0, 8150, 0, 0, 0, 0, 8192, 1, 0, 3449, 124, 8192, 0, 0, 8192, 30, 0, 0, 64, 0, 8179, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16328, 16354, 16250, 9244, 23582, 16524, 16406, 16388] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=115906 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 4, 0, 0, 8192, 0, 0, 0, 8176, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 8175, 4, 0, 2616, 28, 8189, 0, 8174, 0, 0, 0, 0, 8192, 0, 3, 5488, 106, 8191, 0, 0, 8192, 20, 0, 0, 10, 0, 8158, 0, 8192, 0, 0, 0, 5, 0, 8190, 0, 0, 8190, 0, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16390, 16352, 16358, 17658, 15166, 16376, 16390, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=114670 dedup_ratio=0.500092 router_tokens_per_expert=[1, 0, 0, 0, 0, 17, 8188, 0, 0, 8, 8176, 2, 8188, 2, 0, 8184, 4, 4, 8173, 1, 4, 8145, 0, 5, 2794, 122, 8181, 0, 8191, 1, 0, 0, 0, 8184, 7, 0, 4760, 518, 8187, 0, 4, 8187, 36, 1, 4, 0, 0, 8168, 3, 8188, 6, 4, 6, 11, 1, 8186, 0, 6, 8191, 14, 0, 0, 9, 0] payload_rows_per_dst=[16384, 16382, 16364, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16396, 16362, 16308, 5832, 26922, 16418, 16426, 16432] combine_rows_per_src=[16384, 16382, 16364, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114692 dedup_ratio=0.500008 router_tokens_per_expert=[1, 3, 0, 0, 0, 3, 8191, 0, 0, 0, 8185, 0, 8190, 0, 1, 8191, 3, 2, 8190, 1, 1, 8180, 6, 0, 3211, 42, 8185, 0, 8180, 1, 0, 0, 1, 8183, 2, 4, 4746, 120, 8185, 1, 1, 8191, 118, 0, 0, 9, 1, 8167, 1, 8190, 1, 0, 0, 2, 1, 8190, 1, 1, 8187, 0, 0, 0, 1, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16386, 16368, 16382, 22854, 9726, 16590, 16388, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114702 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8184, 1, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 8160, 0, 0, 3814, 73, 8188, 0, 8172, 0, 0, 0, 0, 8191, 1, 0, 4061, 293, 8191, 0, 0, 8192, 9, 0, 0, 17, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 16370, 16320, 24110, 8708, 16412, 16384, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114660 dedup_ratio=0.500015 router_tokens_per_expert=[0, 2, 0, 6, 0, 0, 8192, 0, 0, 0, 8166, 0, 8192, 0, 0, 8191, 0, 0, 8191, 0, 0, 8159, 9, 0, 2914, 128, 8181, 0, 8160, 0, 0, 0, 0, 8192, 0, 8, 5069, 152, 8188, 0, 0, 8189, 41, 0, 0, 37, 0, 8133, 0, 8192, 0, 0, 0, 4, 0, 8192, 0, 0, 8184, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16400, 16332, 16336, 6020, 26812, 16416, 16392, 16368] combine_rows_per_src=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 6, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=106126 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 6, 0, 4, 8192, 0, 0, 0, 8172, 1, 8192, 0, 0, 8192, 4, 0, 8192, 0, 0, 8172, 2, 0, 4227, 88, 8183, 0, 8167, 0, 0, 0, 0, 8192, 0, 4, 3813, 112, 8192, 0, 0, 8187, 23, 0, 0, 24, 0, 8156, 0, 8189, 0, 0, 0, 10, 0, 8188, 0, 0, 8188, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16404, 16346, 16356, 24948, 7858, 16396, 16390, 16376] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 3] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917488 combine_payload_bytes=939491328 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940408816 ingress_bytes=1203732480 link_bytes=1203732480 token_bytes=1879900144 weight_movement_bytes=264241152 total_nvlink_bytes=2144141296 weight_to_token_ratio=0.140561 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740399425e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=922376 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940446472 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879970568 weight_movement_bytes=352321536 total_nvlink_bytes=2232292104 weight_to_token_ratio=0.187408 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=917336 combine_payload_bytes=939327488 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940244824 ingress_bytes=1203568640 link_bytes=1203568640 token_bytes=1879572312 weight_movement_bytes=264241152 total_nvlink_bytes=2143813464 weight_to_token_ratio=0.140586 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739481990e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917512 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425224 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932936 weight_movement_bytes=176160768 total_nvlink_bytes=2056093704 weight_to_token_ratio=0.093706 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917560 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940441656 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965752 weight_movement_bytes=440401920 total_nvlink_bytes=2320367672 weight_to_token_ratio=0.234261 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917376 combine_payload_bytes=939491328 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940408704 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879900032 weight_movement_bytes=528482304 total_nvlink_bytes=2408382336 weight_to_token_ratio=0.281123 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=883256 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940407352 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879931448 weight_movement_bytes=440401920 total_nvlink_bytes=2320333368 weight_to_token_ratio=0.234265 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.024621 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114704 dedup_ratio=0.500069 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 8179, 0, 8192, 0, 0, 8192, 0, 0, 8191, 0, 0, 8158, 7, 0, 2040, 47, 8189, 0, 8165, 0, 0, 0, 0, 8192, 0, 5, 5943, 203, 8190, 0, 0, 8192, 7, 0, 0, 29, 0, 8180, 0, 8191, 0, 0, 0, 2, 0, 8191, 0, 0, 8190, 0, 2, 0, 2, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16368, 16382, 16384] dup_rows_per_dst=[16386, 16592, 16788, 20408, 12298, 15958, 16300, 16360] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16368, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917496 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940294136 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879670776 weight_movement_bytes=176160768 total_nvlink_bytes=2055831544 weight_to_token_ratio=0.093719 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130932 payload_rows_remote=114548 dup_rows=131212 dup_rows_remote=114796 dedup_ratio=0.500534 router_tokens_per_expert=[0, 0, 0, 0, 8186, 0, 0, 8192, 8157, 7743, 0, 0, 0, 8190, 0, 8192, 8192, 0, 41, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8187, 33, 0, 0, 0, 3, 0, 423, 0, 0, 0, 8192, 8192, 0, 0, 8098, 59, 0, 29, 0, 8191, 8157, 39, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16314, 16314, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16372, 14932, 15708, 16536, 16484, 17232, 17532, 16416] combine_rows_per_src=[16384, 16314, 16314, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114680 dedup_ratio=0.500069 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8178, 8183, 8086, 0, 0, 0, 8192, 0, 8192, 8188, 0, 16, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 8192, 8176, 25, 2, 0, 1, 65, 0, 59, 0, 0, 0, 8192, 8192, 0, 0, 8145, 20, 0, 13, 0, 8192, 8167, 20, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16366, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16356, 16172, 16408, 16380, 16410, 16634, 16356, 16374] combine_rows_per_src=[16384, 16366, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130988 payload_rows_remote=114604 dup_rows=131156 dup_rows_remote=114860 dedup_ratio=0.500320 router_tokens_per_expert=[0, 5, 0, 1, 8183, 8, 7, 8158, 8138, 8136, 0, 4, 0, 8183, 3, 8185, 8180, 0, 7, 8183, 5, 5, 2, 8, 0, 0, 0, 12, 1, 4, 1, 8168, 1, 5, 7, 8178, 8171, 63, 0, 0, 48, 38, 4, 109, 0, 0, 0, 8185, 8188, 5, 4, 8112, 1, 9, 20, 1, 8184, 8084, 61, 3, 0, 0, 0, 4] payload_rows_per_dst=[16382, 16302, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16342, 16260, 16396, 16358, 16466, 16750, 16296, 16288] combine_rows_per_src=[16382, 16302, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=114652 dedup_ratio=0.500107 router_tokens_per_expert=[0, 1, 0, 0, 8184, 1, 1, 8187, 8177, 7945, 0, 1, 0, 8183, 0, 8191, 8190, 0, 32, 8191, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 8182, 2, 1, 1, 8187, 8167, 11, 4, 0, 3, 14, 1, 252, 0, 0, 0, 8188, 8191, 0, 1, 8128, 16, 0, 40, 0, 8182, 8175, 33, 1, 0, 0, 1, 1] payload_rows_per_dst=[16384, 16356, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16364, 15890, 16448, 16368, 16362, 16898, 16368, 16402] combine_rows_per_src=[16384, 16356, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114684 dup_rows=131100 dup_rows_remote=115080 dedup_ratio=0.500107 router_tokens_per_expert=[0, 0, 0, 0, 8190, 0, 0, 8189, 8180, 8010, 0, 0, 0, 8191, 0, 8192, 8190, 0, 36, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8192, 8192, 2, 1, 0, 0, 22, 0, 207, 0, 0, 0, 8192, 8192, 0, 0, 8091, 35, 0, 11, 0, 8190, 8176, 15, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16360, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16378, 16020, 16452, 16368, 16390, 16840, 16274, 16378] combine_rows_per_src=[16380, 16360, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=113728 dedup_ratio=0.500099 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8164, 8179, 7603, 0, 0, 0, 8192, 0, 8192, 8181, 0, 329, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 5, 0, 0, 8192, 8128, 35, 1, 0, 3, 60, 0, 430, 0, 0, 0, 8192, 8192, 5, 0, 7897, 74, 1, 33, 0, 8192, 8171, 57, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16358, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16328, 15206, 17020, 16360, 16338, 17370, 16020, 16456] combine_rows_per_src=[16384, 16358, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=114774 dedup_ratio=0.500092 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8187, 8180, 7771, 0, 0, 0, 8191, 0, 8192, 8186, 0, 120, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 3, 0, 0, 8192, 8180, 40, 3, 0, 0, 4, 0, 403, 0, 0, 0, 8192, 8192, 3, 0, 8010, 39, 5, 15, 0, 8192, 8166, 61, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16360, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16374, 15542, 16612, 16322, 16452, 17196, 16144, 16454] combine_rows_per_src=[16384, 16360, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938377216 dispatch_meta_bytes=917376 combine_payload_bytes=938377216 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=939294592 ingress_bytes=1202618368 link_bytes=1202618368 token_bytes=1877671808 weight_movement_bytes=264241152 total_nvlink_bytes=2141912960 weight_to_token_ratio=0.140728 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.734160863e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 4, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917400 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940294040 ingress_bytes=1291698176 link_bytes=1291698176 token_bytes=1879670680 weight_movement_bytes=352321536 total_nvlink_bytes=2231992216 weight_to_token_ratio=0.187438 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232970604e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938835968 dispatch_meta_bytes=917856 combine_payload_bytes=938835968 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939753824 ingress_bytes=1114996736 link_bytes=1114996736 token_bytes=1878589792 weight_movement_bytes=176160768 total_nvlink_bytes=2054750560 weight_to_token_ratio=0.093773 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243516299e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=917248 combine_payload_bytes=939294720 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940211968 ingress_bytes=1467777024 link_bytes=1467777024 token_bytes=1879506688 weight_movement_bytes=528482304 total_nvlink_bytes=2407988992 weight_to_token_ratio=0.281181 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218938654e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=919056 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=940410384 ingress_bytes=1732214784 link_bytes=1732214784 token_bytes=1879901712 weight_movement_bytes=792723456 total_nvlink_bytes=2672625168 weight_to_token_ratio=0.421683 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699679728e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=917752 combine_payload_bytes=939327488 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940245240 ingress_bytes=1291649024 link_bytes=1291649024 token_bytes=1879572728 weight_movement_bytes=352321536 total_nvlink_bytes=2231894264 weight_to_token_ratio=0.187448 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232695374e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=913560 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940224664 ingress_bytes=1115471872 link_bytes=1115471872 token_bytes=1879535768 weight_movement_bytes=176160768 total_nvlink_bytes=2055696536 weight_to_token_ratio=0.093726 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246176862e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114512 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 8188, 0, 0, 8192, 8164, 7931, 0, 0, 0, 8190, 0, 8192, 8186, 0, 25, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 8192, 8168, 22, 4, 0, 2, 11, 0, 413, 0, 0, 0, 8192, 8192, 0, 0, 7952, 45, 0, 44, 0, 8188, 8156, 39, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16564, 21322, 16098, 16380, 16170, 12152, 16082, 16308] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=916784 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940408112 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879899440 weight_movement_bytes=176160768 total_nvlink_bytes=2056060208 weight_to_token_ratio=0.093708 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130994 payload_rows_remote=114610 dup_rows=131150 dup_rows_remote=100814 dedup_ratio=0.500298 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8153, 6210, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 52, 8192, 2745, 0, 684, 8192, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7262, 0, 7663, 0, 8192, 8176, 0, 0, 12, 0] payload_rows_per_dst=[16384, 16306, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16268, 12536, 16316, 16382, 6964, 16458, 15890, 30336] combine_rows_per_src=[16384, 16306, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114658 dup_rows=131102 dup_rows_remote=106610 dedup_ratio=0.500114 router_tokens_per_expert=[0, 8192, 0, 0, 5, 0, 0, 0, 8183, 6850, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 8188, 0, 0, 20, 8192, 737, 3, 3294, 8192, 0, 0, 8192, 8188, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6552, 0, 7148, 0, 8190, 8171, 0, 0, 4, 0] payload_rows_per_dst=[16384, 16366, 16384, 16384, 16384, 16380, 16384, 16376] dup_rows_per_dst=[16394, 13700, 16384, 16382, 24492, 16376, 13104, 14270] combine_rows_per_src=[16384, 16366, 16384, 16384, 16384, 16380, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114668 dup_rows=131098 dup_rows_remote=103692 dedup_ratio=0.500099 router_tokens_per_expert=[6, 8187, 0, 0, 3, 0, 5, 5, 8188, 6902, 8186, 5, 5, 4, 0, 0, 0, 0, 2, 0, 3, 2, 11, 2, 8187, 9, 0, 3, 8, 8171, 0, 0, 55, 8177, 2571, 1, 397, 8188, 13, 4, 8187, 8188, 8183, 5, 0, 4, 4, 0, 0, 8185, 4, 0, 0, 0, 7643, 0, 7030, 4, 8188, 8129, 1, 4, 7, 6] payload_rows_per_dst=[16384, 16378, 16380, 16384, 16380, 16384, 16378, 16378] dup_rows_per_dst=[16400, 13830, 16400, 16372, 9500, 16394, 27406, 14796] combine_rows_per_src=[16384, 16378, 16380, 16384, 16380, 16384, 16378, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=110736 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 2, 0, 0, 0, 8191, 7103, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 1, 0, 8190, 0, 0, 9, 8191, 904, 0, 1367, 8192, 0, 0, 8192, 8184, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7797, 0, 7410, 0, 8192, 8181, 0, 0, 3, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 20336, 16390, 16400, 14792, 16368, 15594, 14804] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114664 dedup_ratio=0.500069 router_tokens_per_expert=[0, 8191, 2, 1, 0, 0, 1, 1, 8184, 7111, 8190, 1, 1, 1, 0, 0, 4, 0, 0, 0, 0, 1, 19, 0, 8191, 1, 0, 5, 0, 8187, 1, 0, 13, 8188, 1998, 0, 1060, 8191, 0, 1, 8190, 8187, 8191, 1, 1, 1, 1, 0, 0, 8190, 1, 0, 0, 0, 7024, 0, 7361, 1, 8190, 8183, 0, 1, 4, 1] payload_rows_per_dst=[16384, 16368, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16388, 14228, 16426, 16386, 22518, 16378, 14048, 14718] combine_rows_per_src=[16384, 16368, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=114708 dedup_ratio=0.500084 router_tokens_per_expert=[0, 8192, 0, 0, 17, 0, 0, 0, 8187, 5733, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 8188, 0, 0, 27, 8192, 3141, 0, 867, 8192, 0, 0, 8192, 8187, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7596, 0, 7210, 0, 8192, 8169, 0, 0, 16, 0] payload_rows_per_dst=[16384, 16374, 16384, 16384, 16384, 16384, 16384, 16372] dup_rows_per_dst=[16418, 11466, 16386, 16386, 24454, 16374, 15192, 14418] combine_rows_per_src=[16384, 16374, 16384, 16384, 16384, 16384, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938885120 dispatch_meta_bytes=861696 combine_payload_bytes=938885120 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939746816 ingress_bytes=1467367424 link_bytes=1467367424 token_bytes=1878631936 weight_movement_bytes=528482304 total_nvlink_bytes=2407114240 weight_to_token_ratio=0.281312 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.018939 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.216645065e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131012 payload_rows_remote=114628 dup_rows=131132 dup_rows_remote=114780 dedup_ratio=0.500229 router_tokens_per_expert=[0, 8192, 0, 0, 18, 0, 0, 0, 8164, 7307, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 8182, 0, 0, 34, 8192, 2105, 5, 869, 8192, 0, 0, 8192, 8176, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7265, 0, 7023, 0, 8192, 8165, 0, 0, 23, 0] payload_rows_per_dst=[16384, 16328, 16384, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16420, 14614, 16384, 16380, 22410, 16352, 14530, 14042] combine_rows_per_src=[16384, 16328, 16384, 16384, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=885072 combine_payload_bytes=939278336 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=940163408 ingress_bytes=1643921408 link_bytes=1643921408 token_bytes=1879441744 weight_movement_bytes=704643072 total_nvlink_bytes=2584084816 weight_to_token_ratio=0.374921 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=27392 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205273678e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=901696 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940425792 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879949888 weight_movement_bytes=352321536 total_nvlink_bytes=2232271424 weight_to_token_ratio=0.187410 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=873440 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940233696 ingress_bytes=1115521024 link_bytes=1115521024 token_bytes=1879593952 weight_movement_bytes=176160768 total_nvlink_bytes=2055754720 weight_to_token_ratio=0.093723 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246452093e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917336 combine_payload_bytes=939376640 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940293976 ingress_bytes=1291698176 link_bytes=1291698176 token_bytes=1879670616 weight_movement_bytes=352321536 total_nvlink_bytes=2231992152 weight_to_token_ratio=0.187438 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232970604e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=917496 combine_payload_bytes=939343872 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940261368 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879605240 weight_movement_bytes=176160768 total_nvlink_bytes=2055766008 weight_to_token_ratio=0.093722 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=114704 dedup_ratio=0.500107 router_tokens_per_expert=[0, 8192, 0, 0, 6, 0, 0, 0, 8169, 7104, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 1, 0, 8191, 0, 0, 37, 8192, 1638, 0, 1298, 8192, 0, 0, 8192, 8184, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 7657, 0, 6885, 0, 8191, 8159, 0, 0, 12, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16358] dup_rows_per_dst=[16396, 30546, 16390, 16384, 5946, 16378, 15316, 13744] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=917632 combine_payload_bytes=939032576 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=939950208 ingress_bytes=1643675648 link_bytes=1643675648 token_bytes=1878982784 weight_movement_bytes=704643072 total_nvlink_bytes=2583625856 weight_to_token_ratio=0.375013 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.203897525e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=917456 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940212176 ingress_bytes=1115455488 link_bytes=1115455488 token_bytes=1879506896 weight_movement_bytes=176160768 total_nvlink_bytes=2055667664 weight_to_token_ratio=0.093727 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246085119e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=113480 dedup_ratio=0.500023 router_tokens_per_expert=[0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 428, 8192, 0, 0, 0, 8110, 0, 3, 0, 8192, 0, 8192, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 8192, 8189, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 7270, 0, 0, 8192, 0, 0, 0, 0, 415, 0, 192, 0, 0] payload_rows_per_dst=[16384, 16378, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16376, 3728, 16226, 13694, 16378, 16414, 30664, 17598] combine_rows_per_src=[16384, 16378, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114696 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 1522, 8192, 0, 0, 0, 8132, 0, 4, 0, 8192, 0, 8192, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 8192, 8188, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 6582, 0, 0, 8192, 0, 0, 0, 0, 157, 0, 96, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 19230, 16272, 16372, 16376, 16384, 13164, 16890] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=112636 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 1087, 8188, 0, 0, 0, 8116, 0, 7, 0, 8192, 0, 8192, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 1, 8192, 8192, 8192, 0, 0, 0, 8192, 8191, 0, 8192, 1, 0, 0, 0, 6911, 0, 0, 8192, 0, 0, 0, 0, 272, 0, 60, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 18436, 16238, 16374, 16386, 16384, 13822, 17048] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 2, 2, 0, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=112812 dedup_ratio=0.500053 router_tokens_per_expert=[0, 0, 8185, 4, 0, 3, 0, 0, 4, 0, 0, 5, 8076, 5, 13, 1292, 8173, 6, 0, 0, 8133, 7, 18, 5, 8187, 6, 8185, 0, 6, 0, 3, 8187, 8179, 4, 1, 3, 1, 8183, 8177, 8186, 6, 1, 6, 8185, 8185, 7, 8186, 1, 8, 1, 0, 6463, 0, 3, 8186, 5, 0, 6, 3, 539, 0, 44, 0, 0] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16376, 16384, 16384, 16384] dup_rows_per_dst=[16378, 17870, 16300, 11974, 16354, 16398, 18274, 17538] combine_rows_per_src=[16378, 16384, 16384, 16384, 16376, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114782 dedup_ratio=0.500000 router_tokens_per_expert=[0, 1, 8191, 1, 0, 1, 0, 0, 0, 0, 0, 1, 8107, 0, 0, 513, 8190, 1, 0, 0, 8140, 0, 5, 1, 8191, 1, 8191, 1, 1, 0, 0, 8180, 8191, 0, 0, 1, 2, 8191, 8182, 8191, 1, 1, 0, 8191, 8189, 1, 8191, 4, 1, 1, 0, 6945, 0, 0, 8191, 1, 0, 1, 0, 862, 1, 16, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 17240, 16290, 16364, 16368, 16390, 13894, 18140] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114702 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 1099, 8187, 0, 0, 0, 8105, 0, 6, 0, 8192, 0, 8192, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 8192, 8161, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 6369, 0, 0, 8192, 0, 0, 0, 0, 748, 1, 141, 0, 4] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 18490, 16212, 16370, 16322, 16384, 12738, 18172] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114690 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 1439, 8192, 0, 0, 0, 8078, 0, 16, 0, 8192, 0, 8192, 0, 0, 0, 0, 8169, 8192, 0, 0, 0, 0, 8192, 8172, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 6498, 0, 0, 8191, 0, 0, 0, 0, 212, 0, 237, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16384, 19156, 16188, 16338, 16344, 16384, 12996, 17284] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=912648 combine_payload_bytes=939474944 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940387592 ingress_bytes=1291796480 link_bytes=1291796480 token_bytes=1879862536 weight_movement_bytes=352321536 total_nvlink_bytes=2232184072 weight_to_token_ratio=0.187419 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233521066e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917536 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940441632 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879965728 weight_movement_bytes=880803840 total_nvlink_bytes=2760769568 weight_to_token_ratio=0.468521 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.018939 gemm_active_groups=16 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[3, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=909296 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940433392 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879957488 weight_movement_bytes=176160768 total_nvlink_bytes=2056118256 weight_to_token_ratio=0.093705 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=909944 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940319352 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879728760 weight_movement_bytes=176160768 total_nvlink_bytes=2055889528 weight_to_token_ratio=0.093716 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917880 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441976 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879966072 weight_movement_bytes=176160768 total_nvlink_bytes=2056126840 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917560 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441656 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965752 weight_movement_bytes=528482304 total_nvlink_bytes=2408448056 weight_to_token_ratio=0.281113 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114672 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 554, 8189, 0, 0, 0, 8074, 0, 11, 0, 8192, 0, 8192, 0, 0, 0, 0, 8178, 8192, 0, 0, 0, 0, 8192, 8183, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 7761, 0, 0, 8192, 0, 0, 0, 0, 226, 0, 62, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16402, 16928, 17346, 23586, 16552, 16336, 15522, 8402] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=940425216 ingress_bytes=1379909632 link_bytes=1379909632 token_bytes=1879932928 weight_movement_bytes=440401920 total_nvlink_bytes=2320334848 weight_to_token_ratio=0.234265 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726917937e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 6, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114686 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 4604, 0, 0, 8066, 4, 1, 8192, 8192, 0, 0, 0, 0, 8175, 0, 0, 3716, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 12, 0, 0, 8159, 0, 0, 0, 8191, 0, 8192, 0, 0, 37, 0, 8192, 8192, 8186, 0, 0, 8192, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8956, 16300, 23484, 16370, 16322, 16270, 16984, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917432 combine_payload_bytes=939507712 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940425144 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932856 weight_movement_bytes=264241152 total_nvlink_bytes=2144174008 weight_to_token_ratio=0.140559 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114670 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 2368, 0, 3, 8126, 0, 0, 8192, 8192, 0, 0, 0, 0, 8181, 0, 0, 5909, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 15, 0, 0, 8186, 0, 0, 0, 8192, 5, 8181, 0, 0, 8, 0, 8192, 8192, 8169, 0, 0, 8192, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8516, 16384, 24274, 16384, 16402, 16388, 16340, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114672 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 5091, 0, 0, 8144, 5, 3, 8192, 8192, 0, 0, 0, 0, 8182, 0, 0, 3170, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 10, 0, 0, 8192, 0, 0, 0, 8192, 0, 8169, 0, 0, 23, 0, 8192, 8192, 8157, 0, 0, 8192, 8192, 0, 0, 0, 0, 1, 0, 5, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[26470, 16400, 6320, 16384, 16404, 16384, 16314, 16396] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114684 dup_rows=131090 dup_rows_remote=114766 dedup_ratio=0.500069 router_tokens_per_expert=[7, 0, 8, 6, 2896, 5, 0, 8133, 11, 6, 8186, 8186, 8, 0, 0, 6, 8156, 0, 5, 5300, 0, 2, 8184, 0, 0, 0, 11, 2, 0, 8185, 5, 0, 4, 3, 42, 5, 5, 8167, 1, 0, 0, 8186, 0, 8118, 7, 1, 92, 6, 8173, 8184, 8168, 5, 0, 8186, 8174, 0, 0, 5, 4, 14, 12, 19, 0, 8183] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16370, 16382] dup_rows_per_dst=[5726, 16422, 26912, 16390, 16442, 16436, 16324, 16438] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16370, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=124660 dedup_ratio=0.500038 router_tokens_per_expert=[1, 0, 1, 1, 5115, 0, 0, 8126, 10, 4, 8191, 8184, 1, 0, 4, 1, 8185, 0, 1, 3217, 0, 0, 8191, 0, 0, 0, 4, 0, 0, 8191, 1, 0, 1, 0, 11, 1, 1, 8186, 0, 0, 1, 8191, 6, 8173, 0, 1, 19, 1, 8189, 8191, 8090, 1, 0, 8190, 8187, 1, 0, 1, 1, 2, 0, 7, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16376, 16384] dup_rows_per_dst=[26486, 16406, 6422, 16390, 16398, 16400, 16182, 16398] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 4744, 0, 1, 8120, 0, 0, 8192, 8188, 0, 0, 0, 0, 8151, 0, 0, 3572, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 12, 0, 0, 8188, 0, 0, 0, 8192, 4, 8174, 0, 0, 29, 0, 8192, 8192, 8159, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[22096, 16376, 10696, 16384, 16400, 16414, 16318, 16388] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917496 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441592 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965688 weight_movement_bytes=176160768 total_nvlink_bytes=2056126456 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114612 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 4935, 0, 7, 8137, 0, 0, 8192, 8185, 0, 0, 1, 0, 8154, 0, 0, 3372, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 28, 0, 0, 8125, 0, 0, 0, 8192, 9, 8189, 0, 0, 33, 0, 8192, 8192, 8162, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 8192] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[9776, 16372, 23052, 16384, 16306, 16462, 16324, 16398] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917432 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441528 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965624 weight_movement_bytes=176160768 total_nvlink_bytes=2056126392 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917440 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940441536 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965632 weight_movement_bytes=264241152 total_nvlink_bytes=2144206784 weight_to_token_ratio=0.140556 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917800 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940409128 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=1879900456 weight_movement_bytes=704643072 total_nvlink_bytes=2584543528 weight_to_token_ratio=0.374830 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=957352 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940399528 ingress_bytes=1379844096 link_bytes=1379844096 token_bytes=1879841704 weight_movement_bytes=440401920 total_nvlink_bytes=2320243624 weight_to_token_ratio=0.234276 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726550962e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=108024 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 3443, 0, 0, 8081, 1, 0, 8192, 8189, 0, 0, 2, 0, 8160, 0, 0, 4924, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 17, 0, 0, 8183, 0, 0, 0, 8192, 4, 8192, 0, 0, 42, 0, 8192, 8192, 8100, 0, 0, 8192, 8188, 0, 0, 0, 0, 1, 0, 8, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[23048, 16412, 9914, 16386, 16400, 16318, 16308, 16286] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917192 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940424904 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932616 weight_movement_bytes=352321536 total_nvlink_bytes=2232254152 weight_to_token_ratio=0.187412 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.017992 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130382 payload_rows_remote=113998 dup_rows=131762 dup_rows_remote=114364 dedup_ratio=0.502632 router_tokens_per_expert=[0, 0, 0, 8192, 3906, 0, 0, 8192, 0, 8192, 0, 0, 0, 990, 0, 24, 8192, 0, 1, 0, 0, 0, 1, 8192, 8192, 8191, 3402, 8192, 0, 8192, 8192, 0, 8154, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8098, 0, 0] payload_rows_per_dst=[15694, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[7348, 7088, 16388, 34168, 16652, 16384, 16336, 17398] combine_rows_per_src=[15694, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=890848 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940414944 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879939040 weight_movement_bytes=176160768 total_nvlink_bytes=2056099808 weight_to_token_ratio=0.093706 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.018939 gemm_active_groups=17 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113042 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 2072, 0, 0, 8192, 0, 8192, 0, 0, 0, 818, 0, 5, 8192, 0, 0, 0, 0, 0, 2, 8192, 8192, 8191, 5332, 8192, 0, 8192, 8192, 0, 8177, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8160, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[20528, 18030, 16388, 10664, 16374, 16384, 16384, 16320] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114700 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 2188, 0, 1, 8192, 0, 8188, 0, 0, 0, 413, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 8192, 5633, 8192, 0, 8192, 8192, 0, 8177, 0, 8, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 8192, 0, 3, 8146, 3, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17388, 17212, 16384, 14640, 16372, 16386, 16386, 16304] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114688 dup_rows=131076 dup_rows_remote=124678 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 8192, 4563, 0, 0, 8192, 0, 8186, 0, 0, 0, 504, 0, 12, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 8192, 3200, 8192, 0, 8192, 8189, 0, 8156, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8128, 9, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[25510, 17404, 16384, 6398, 16338, 16384, 16384, 16274] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114664 dedup_ratio=0.500023 router_tokens_per_expert=[8, 0, 3, 8186, 3134, 6, 0, 8185, 6, 8187, 7, 7, 0, 749, 6, 0, 8186, 0, 4, 0, 7, 24, 1, 8175, 8180, 8184, 4303, 8186, 6, 8183, 8186, 0, 8147, 5, 5, 0, 6, 0, 0, 0, 0, 0, 11, 0, 6, 5, 0, 7, 8182, 0, 7, 0, 16, 6, 0, 3, 0, 8185, 8185, 1, 0, 8186, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16378] dup_rows_per_dst=[6288, 17900, 16410, 24978, 16312, 16410, 16414, 16366] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114698 dedup_ratio=0.500031 router_tokens_per_expert=[1, 0, 0, 8191, 2192, 1, 4, 8191, 1, 8185, 1, 1, 0, 1101, 1, 2, 8191, 0, 0, 0, 1, 1, 0, 8190, 8189, 8191, 4931, 8191, 1, 8191, 8184, 0, 8178, 0, 10, 11, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8189, 1, 1, 0, 6, 1, 0, 1, 0, 8191, 8191, 1, 1, 8154, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16384, 16384, 16384, 16384] dup_rows_per_dst=[20776, 18578, 16382, 9856, 16398, 16384, 16396, 16310] combine_rows_per_src=[16384, 16384, 16384, 16376, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=933871616 dispatch_meta_bytes=913448 combine_payload_bytes=933871616 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=934785064 ingress_bytes=1286193152 link_bytes=1286193152 token_bytes=1868656680 weight_movement_bytes=352321536 total_nvlink_bytes=2220978216 weight_to_token_ratio=0.188543 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.202144768e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 3246, 0, 2, 8192, 0, 8180, 0, 0, 0, 507, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 8192, 4746, 8192, 0, 8192, 8192, 0, 8124, 0, 20, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 8192, 0, 0, 7929, 8, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[10676, 17378, 16384, 21696, 16290, 16388, 16386, 15874] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=910920 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940435016 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879959112 weight_movement_bytes=176160768 total_nvlink_bytes=2056119880 weight_to_token_ratio=0.093705 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917552 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441648 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965744 weight_movement_bytes=352321536 total_nvlink_bytes=2232287280 weight_to_token_ratio=0.187408 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 4, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=957464 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=940481560 ingress_bytes=1996488704 link_bytes=1996488704 token_bytes=1880005656 weight_movement_bytes=1056964608 total_nvlink_bytes=2936970264 weight_to_token_ratio=0.562214 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117950337e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=917384 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940392328 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879867272 weight_movement_bytes=176160768 total_nvlink_bytes=2056028040 weight_to_token_ratio=0.093709 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=107826 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 8192, 3411, 0, 0, 8192, 0, 8181, 0, 0, 0, 550, 0, 10, 8192, 0, 0, 0, 0, 6, 0, 8191, 8192, 8192, 4323, 8192, 0, 8192, 8191, 0, 8152, 0, 14, 1, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 8192, 0, 0, 8116, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[23248, 17482, 16354, 8684, 16336, 16352, 16386, 16232] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917512 combine_payload_bytes=939458560 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940376072 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834632 weight_movement_bytes=176160768 total_nvlink_bytes=2055995400 weight_to_token_ratio=0.093711 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441584 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965680 weight_movement_bytes=352321536 total_nvlink_bytes=2232287216 weight_to_token_ratio=0.187408 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114650 dup_rows=131110 dup_rows_remote=113470 dedup_ratio=0.500145 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 5888, 0, 178, 8192, 0, 0, 0, 0, 9, 0, 7, 0, 0, 0, 0, 0, 7656, 8173, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 2485, 8192, 8157, 0, 0, 8192, 0, 0, 3, 0, 0, 25, 0, 8189, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 190, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16346, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16282, 12132, 16106, 15312, 20510, 16440, 16688, 17640] combine_rows_per_src=[16384, 16384, 16384, 16346, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=890048 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940397760 ingress_bytes=1467990016 link_bytes=1467990016 token_bytes=1879905472 weight_movement_bytes=528482304 total_nvlink_bytes=2408387776 weight_to_token_ratio=0.281122 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220131320e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131036 payload_rows_remote=114652 dup_rows=131108 dup_rows_remote=108206 dedup_ratio=0.500137 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 4851, 0, 84, 8192, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 8099, 8177, 0, 0, 0, 10, 8192, 0, 0, 8192, 0, 3297, 8192, 8154, 0, 0, 8192, 0, 0, 0, 0, 0, 11, 0, 8186, 0, 8192, 8192, 0, 0, 0, 8188, 0, 0, 8192, 13, 0, 72, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16356, 16384, 16384, 16376, 16384] dup_rows_per_dst=[16384, 9870, 16404, 16216, 22902, 16406, 16372, 16554] combine_rows_per_src=[16384, 16384, 16384, 16356, 16384, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114638 dup_rows=131122 dup_rows_remote=103680 dedup_ratio=0.500191 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 6036, 0, 85, 8192, 0, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 8103, 8176, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 2108, 8192, 8175, 0, 0, 8192, 0, 1, 0, 0, 0, 8, 0, 8187, 0, 8192, 8191, 0, 0, 0, 8192, 0, 0, 8192, 8, 0, 62, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16356, 16362, 16384, 16384, 16384] dup_rows_per_dst=[16384, 27442, 16402, 16206, 5388, 16402, 16374, 16524] combine_rows_per_src=[16384, 16384, 16384, 16356, 16362, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[2, 2, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131010 payload_rows_remote=114684 dup_rows=131134 dup_rows_remote=115350 dedup_ratio=0.500237 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 1, 0, 5444, 0, 156, 8192, 0, 0, 0, 0, 11, 0, 13, 0, 0, 0, 0, 0, 7884, 8163, 0, 0, 0, 8, 8192, 0, 0, 8192, 0, 2842, 8192, 8171, 0, 0, 8192, 0, 0, 0, 0, 1, 21, 0, 8185, 0, 8192, 8192, 0, 0, 0, 8190, 0, 0, 8192, 12, 0, 50, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16326, 16384, 16384, 16380, 16384] dup_rows_per_dst=[16386, 11200, 16432, 15784, 22026, 16428, 16370, 16508] combine_rows_per_src=[16384, 16384, 16384, 16326, 16384, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=114794 dedup_ratio=0.500092 router_tokens_per_expert=[0, 0, 0, 8179, 1, 0, 0, 58, 5223, 0, 73, 8178, 0, 6, 7, 0, 0, 7, 2, 8, 0, 3, 0, 4, 8037, 8183, 6, 6, 1, 425, 8185, 9, 1, 8186, 0, 2792, 8177, 7946, 6, 8, 8186, 11, 17, 2, 0, 0, 10, 7, 8138, 7, 8181, 8184, 6, 0, 6, 8186, 25, 6, 8184, 10, 0, 5, 0, 8184] payload_rows_per_dst=[16378, 16380, 16384, 16380, 16374, 16384, 16384, 16384] dup_rows_per_dst=[16470, 10594, 16394, 16954, 21486, 16452, 16302, 16444] combine_rows_per_src=[16378, 16380, 16384, 16380, 16374, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114658 dup_rows=131102 dup_rows_remote=114706 dedup_ratio=0.500114 router_tokens_per_expert=[1, 0, 0, 8187, 1, 0, 6, 4, 5456, 0, 86, 8190, 0, 1, 1, 0, 5, 1, 6, 1, 0, 1, 0, 1, 8087, 8184, 1, 1, 1, 144, 8191, 2, 1, 8191, 0, 2706, 8189, 8048, 0, 1, 8190, 5, 0, 0, 0, 0, 3, 0, 8185, 1, 8186, 8189, 1, 0, 0, 8190, 2, 1, 8187, 4, 0, 42, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16372, 16370, 16382, 16382, 16384] dup_rows_per_dst=[16396, 17944, 16396, 16468, 14660, 16396, 16372, 16470] combine_rows_per_src=[16384, 16384, 16384, 16372, 16370, 16382, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=891432 combine_payload_bytes=939229184 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940120616 ingress_bytes=1555791872 link_bytes=1555791872 token_bytes=1879349800 weight_movement_bytes=616562688 total_nvlink_bytes=2495912488 weight_to_token_ratio=0.328072 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.711785064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=912480 combine_payload_bytes=939212800 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940125280 ingress_bytes=1379614720 link_bytes=1379614720 token_bytes=1879338080 weight_movement_bytes=440401920 total_nvlink_bytes=2319740000 weight_to_token_ratio=0.234339 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725266552e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131006 payload_rows_remote=114622 dup_rows=131138 dup_rows_remote=114712 dedup_ratio=0.500252 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 1, 0, 5960, 0, 178, 8192, 0, 0, 0, 0, 13, 0, 24, 0, 0, 0, 0, 0, 8007, 8165, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 2158, 8192, 8107, 0, 0, 8192, 0, 0, 0, 0, 0, 21, 0, 8171, 0, 8192, 8192, 0, 0, 0, 8186, 0, 0, 8190, 43, 0, 118, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16330, 16384, 16384, 16372, 16384] dup_rows_per_dst=[16386, 12276, 16458, 16018, 20530, 16426, 16342, 16702] combine_rows_per_src=[16384, 16384, 16384, 16330, 16384, 16384, 16372, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=873272 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939987768 ingress_bytes=1115275264 link_bytes=1115275264 token_bytes=1879102264 weight_movement_bytes=176160768 total_nvlink_bytes=2055263032 weight_to_token_ratio=0.093747 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245075940e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 4, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=920136 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940411464 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879902792 weight_movement_bytes=352321536 total_nvlink_bytes=2232224328 weight_to_token_ratio=0.187415 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=917832 combine_payload_bytes=939327488 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940245320 ingress_bytes=1467809792 link_bytes=1467809792 token_bytes=1879572808 weight_movement_bytes=528482304 total_nvlink_bytes=2408055112 weight_to_token_ratio=0.281171 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219122141e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130918 payload_rows_remote=114534 dup_rows=131226 dup_rows_remote=114836 dedup_ratio=0.500587 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 3, 0, 6493, 0, 124, 8192, 0, 0, 0, 0, 25, 0, 24, 0, 0, 0, 0, 0, 8065, 8173, 0, 0, 0, 1, 8192, 0, 0, 8192, 0, 1653, 8192, 8108, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 8165, 0, 8192, 8191, 0, 0, 0, 8190, 0, 0, 8192, 19, 0, 109, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16330, 16290] dup_rows_per_dst=[16390, 29618, 16480, 18336, 3622, 16124, 16332, 14324] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16330, 16290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=917456 combine_payload_bytes=939278336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940195792 ingress_bytes=1291599872 link_bytes=1291599872 token_bytes=1879474128 weight_movement_bytes=352321536 total_nvlink_bytes=2231795664 weight_to_token_ratio=0.187458 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232420143e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=917336 combine_payload_bytes=938983424 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939900760 ingress_bytes=1115144192 link_bytes=1115144192 token_bytes=1878884184 weight_movement_bytes=176160768 total_nvlink_bytes=2055044952 weight_to_token_ratio=0.093758 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244341991e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938262528 dispatch_meta_bytes=917480 combine_payload_bytes=938262528 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939180008 ingress_bytes=1114423296 link_bytes=1114423296 token_bytes=1877442536 weight_movement_bytes=176160768 total_nvlink_bytes=2053603304 weight_to_token_ratio=0.093830 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.240305274e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=105988 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 4130, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8124, 0, 0, 0, 0, 0, 8185, 0, 8160, 0, 0, 26, 0, 0, 0, 8186, 0, 0, 0, 8179, 0, 21, 0, 7, 0, 0, 8192, 4134, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8260, 16356, 16360, 16384, 16234, 16388, 16006, 25084] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114740 dedup_ratio=0.500008 router_tokens_per_expert=[8192, 1857, 8184, 8192, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8192, 0, 8169, 0, 0, 0, 0, 0, 8190, 0, 8178, 0, 0, 11, 0, 0, 0, 8187, 0, 0, 3, 8183, 0, 6, 0, 11, 0, 0, 8192, 6363, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[13324, 16380, 16384, 16392, 16334, 16370, 16406, 19484] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 3119, 8186, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8172, 0, 0, 0, 0, 0, 8178, 0, 8183, 0, 0, 10, 0, 0, 0, 8187, 0, 0, 0, 8186, 0, 8, 0, 3, 0, 0, 8192, 5112, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[22610, 16384, 16384, 16384, 16316, 16376, 16394, 10224] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114692 dedup_ratio=0.500053 router_tokens_per_expert=[8192, 1345, 8186, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 8157, 0, 0, 0, 0, 0, 8189, 0, 8161, 0, 0, 2, 0, 0, 0, 8184, 0, 0, 5, 8188, 0, 17, 0, 10, 0, 0, 8192, 6897, 8192, 1, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16372, 16384, 16384] dup_rows_per_dst=[19062, 16378, 16384, 16394, 16310, 16322, 16440, 13796] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114590 dedup_ratio=0.500069 router_tokens_per_expert=[8185, 3247, 8186, 8184, 4, 4, 14, 6, 8, 8186, 0, 6, 0, 6, 18, 8167, 8181, 0, 7, 7, 9, 1, 5, 0, 3, 4, 7, 0, 6, 0, 8175, 0, 8129, 4, 997, 18, 6, 0, 8160, 0, 8117, 0, 6, 303, 0, 7, 0, 8180, 3, 11, 0, 8179, 5, 47, 10, 3, 0, 4, 8177, 3712, 8184, 0, 0, 8184] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16376] dup_rows_per_dst=[6538, 16398, 16404, 16376, 18244, 16852, 16500, 23778] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114702 dedup_ratio=0.500069 router_tokens_per_expert=[8191, 4780, 8182, 8191, 1, 0, 1, 1, 0, 8190, 1, 1, 0, 10, 3, 8190, 8191, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 6, 8188, 0, 8170, 1, 259, 3, 1, 0, 8188, 0, 8175, 0, 1, 106, 0, 1, 0, 8185, 0, 2, 4, 8187, 0, 9, 0, 14, 0, 1, 8190, 3055, 8191, 4, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16372, 16384, 16384] dup_rows_per_dst=[25928, 16406, 16388, 16392, 16866, 16564, 16430, 6116] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=882704 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940406800 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879930896 weight_movement_bytes=528482304 total_nvlink_bytes=2408413200 weight_to_token_ratio=0.281118 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917704 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940425416 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879933128 weight_movement_bytes=352321536 total_nvlink_bytes=2232254664 weight_to_token_ratio=0.187412 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114688 dup_rows=131094 dup_rows_remote=114810 dedup_ratio=0.500084 router_tokens_per_expert=[8192, 4194, 8171, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8192, 0, 8160, 0, 3, 0, 0, 0, 8191, 0, 8141, 0, 0, 8, 0, 0, 0, 8174, 0, 0, 3, 8180, 0, 27, 0, 26, 0, 0, 8192, 4049, 8192, 1, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16362, 16384, 16384] dup_rows_per_dst=[8346, 16384, 16384, 16416, 16324, 16284, 16472, 24484] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16362, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131032 payload_rows_remote=114648 dup_rows=131112 dup_rows_remote=104108 dedup_ratio=0.500153 router_tokens_per_expert=[8192, 4174, 8168, 8192, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8192, 0, 8177, 0, 1, 0, 0, 0, 8182, 0, 8168, 0, 0, 8, 0, 0, 0, 8187, 0, 0, 2, 8183, 0, 19, 0, 8, 0, 0, 8192, 4057, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16376, 16352, 16384, 16384] dup_rows_per_dst=[27004, 16386, 16384, 16334, 14460, 16006, 16424, 8114] combine_rows_per_src=[16384, 16384, 16384, 16384, 16376, 16352, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917464 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940326872 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879736280 weight_movement_bytes=176160768 total_nvlink_bytes=2055897048 weight_to_token_ratio=0.093716 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917040 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940293680 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879670320 weight_movement_bytes=176160768 total_nvlink_bytes=2055831088 weight_to_token_ratio=0.093719 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917488 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940294128 ingress_bytes=1115537408 link_bytes=1115537408 token_bytes=1879670768 weight_movement_bytes=176160768 total_nvlink_bytes=2055831536 weight_to_token_ratio=0.093719 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246543837e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917992 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940442088 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879966184 weight_movement_bytes=352321536 total_nvlink_bytes=2232287720 weight_to_token_ratio=0.187408 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=875024 combine_payload_bytes=939196416 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940071440 ingress_bytes=1467678720 link_bytes=1467678720 token_bytes=1879267856 weight_movement_bytes=528482304 total_nvlink_bytes=2407750160 weight_to_token_ratio=0.281217 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218388193e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8188, 8150, 1771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 8192, 8192, 0, 8189, 1, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 14, 0, 2641, 8192, 8192, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16300, 19926, 16376, 16386, 16384, 21694, 7654, 16352] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114348 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 8024, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 8192, 8190, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 24, 0, 4554, 8192, 8192, 0, 0, 0, 2540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16036, 2550, 16396, 16182, 16180, 9156, 37848, 16724] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=110320 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 8131, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 8192, 8190, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 13, 0, 2669, 8192, 8192, 0, 0, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16262, 20752, 16384, 16384, 16380, 21746, 6806, 16358] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114680 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8188, 8051, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 8192, 8192, 0, 8192, 0, 0, 8192, 8192, 8192, 0, 2, 0, 0, 0, 0, 8192, 8188, 0, 0, 10, 0, 3321, 8192, 8192, 0, 0, 0, 3366, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8172] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16102, 19696, 16376, 16392, 16388, 23038, 6732, 16348] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441600 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965696 weight_movement_bytes=528482304 total_nvlink_bytes=2408448000 weight_to_token_ratio=0.281113 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916144 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940440240 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879964336 weight_movement_bytes=352321536 total_nvlink_bytes=2232285872 weight_to_token_ratio=0.187409 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=98510 dedup_ratio=0.500038 router_tokens_per_expert=[8052, 0, 1, 6, 4, 0, 8176, 8187, 2648, 7, 1, 0, 0, 0, 7, 6, 0, 5, 0, 12, 6, 58, 1, 8186, 0, 3, 8181, 8184, 0, 8145, 0, 17, 8183, 8185, 8174, 0, 0, 0, 0, 7, 7, 8184, 8160, 15, 0, 51, 0, 1813, 8171, 8183, 0, 27, 0, 3863, 5, 0, 0, 7, 8, 0, 9, 7, 4, 8116] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16116, 13226, 16504, 16278, 16392, 3708, 32572, 16286] combine_rows_per_src=[16384, 16384, 16384, 16378, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=900032 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940424128 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879948224 weight_movement_bytes=352321536 total_nvlink_bytes=2232269760 weight_to_token_ratio=0.187410 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=119650 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8180, 7909, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 8192, 8192, 0, 8191, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 30, 0, 3659, 8192, 8192, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8157] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15818, 18032, 16360, 16408, 16384, 11422, 20330, 16318] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114680 dedup_ratio=0.500008 router_tokens_per_expert=[8159, 0, 7, 1, 1, 0, 8187, 8119, 887, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 8, 1, 8191, 4, 1, 8189, 8189, 0, 8184, 2, 2, 8191, 8191, 8185, 0, 0, 0, 0, 1, 1, 8191, 8190, 3, 0, 8, 0, 2144, 8190, 8191, 0, 0, 0, 5270, 1, 0, 0, 1, 1, 0, 1, 1, 0, 8173] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16190, 18154, 16394, 16384, 16370, 20690, 10540, 16352] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=112824 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8188, 8100, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 8192, 8192, 0, 8191, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8187, 0, 0, 21, 0, 2821, 8192, 8192, 0, 0, 0, 4296, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8163] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[18248, 18736, 16282, 16664, 16598, 19618, 8592, 16334] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917472 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441568 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965664 weight_movement_bytes=528482304 total_nvlink_bytes=2408447968 weight_to_token_ratio=0.281113 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 3, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=852752 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940294928 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879737104 weight_movement_bytes=352321536 total_nvlink_bytes=2232058640 weight_to_token_ratio=0.187431 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=937352 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=940461448 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879985544 weight_movement_bytes=528482304 total_nvlink_bytes=2408467848 weight_to_token_ratio=0.281110 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.024621 gemm_active_groups=14 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917464 combine_payload_bytes=939507712 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940425176 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932888 weight_movement_bytes=264241152 total_nvlink_bytes=2144174040 weight_to_token_ratio=0.140559 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=910048 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940434144 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879958240 weight_movement_bytes=440401920 total_nvlink_bytes=2320360160 weight_to_token_ratio=0.234262 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8127, 0, 0, 0, 0, 8189, 8167, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 7828, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 3, 8192, 445, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16204, 16384, 15656, 16384, 17274, 16396, 16390] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114658 dedup_ratio=0.500008 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 8168, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8063, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 156, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 7162, 16352, 25294, 16384, 16696, 16386, 16416] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=114792 dedup_ratio=0.500008 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8190, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 8147, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8030, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 213, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16282, 16384, 16060, 16384, 16810, 16390, 16384] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 0, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114900 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 8188, 8176, 0, 8192, 8192, 0, 0, 0, 1, 0, 0, 8192, 8086, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 3, 8192, 123, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16340, 16386, 16172, 16384, 16628, 16386, 16392] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917376 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=940425088 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932800 weight_movement_bytes=352321536 total_nvlink_bytes=2232254336 weight_to_token_ratio=0.187412 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917920 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940442016 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879966112 weight_movement_bytes=704643072 total_nvlink_bytes=2584609184 weight_to_token_ratio=0.374817 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.023674 gemm_active_groups=12 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130986 payload_rows_remote=114602 dup_rows=131158 dup_rows_remote=113278 dedup_ratio=0.500328 router_tokens_per_expert=[0, 8143, 0, 15, 7, 7694, 4, 0, 0, 8175, 0, 6, 0, 0, 8183, 8180, 28, 8182, 8184, 0, 2, 0, 4, 0, 8, 8183, 5719, 0, 6, 10, 8181, 8184, 0, 8, 0, 10, 5, 0, 0, 4, 5, 8184, 8160, 0, 8182, 2226, 6, 0, 475, 0, 0, 9, 2, 4, 276, 0, 0, 1, 8185, 8, 8174, 0, 45, 5] payload_rows_per_dst=[16316, 16382, 16376, 16382, 16384, 16378, 16384, 16384] dup_rows_per_dst=[15410, 16340, 16424, 11466, 16404, 20782, 17880, 16452] combine_rows_per_src=[16316, 16382, 16376, 16382, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113894 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8189, 0, 0, 0, 8153, 0, 0, 0, 0, 8185, 8154, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 7853, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 3, 8192, 401, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16378, 16230, 16384, 15706, 16384, 17178, 16412, 16400] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=113406 dedup_ratio=0.500046 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8191, 0, 0, 0, 8175, 0, 0, 0, 0, 8187, 8112, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8103, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 2, 8192, 180, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16372] dup_rows_per_dst=[17678, 26244, 16404, 16230, 16360, 7444, 14492, 16232] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114684 dup_rows=131080 dup_rows_remote=114714 dedup_ratio=0.500031 router_tokens_per_expert=[0, 8189, 0, 7, 1, 8065, 1, 0, 1, 8171, 0, 0, 0, 0, 8186, 8158, 4, 8185, 8181, 0, 0, 0, 3, 0, 1, 8191, 7243, 0, 1, 2, 8190, 8191, 0, 1, 0, 1, 1, 0, 1, 0, 1, 8191, 8188, 3, 8191, 946, 1, 0, 116, 0, 0, 1, 0, 1, 57, 0, 4, 1, 8191, 2, 8186, 0, 16, 1] payload_rows_per_dst=[16384, 16380, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16142, 16280, 16366, 14490, 16388, 18266, 16730, 16418] combine_rows_per_src=[16384, 16380, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918352 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940442448 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879966544 weight_movement_bytes=528482304 total_nvlink_bytes=2408448848 weight_to_token_ratio=0.281113 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938819584 dispatch_meta_bytes=911520 combine_payload_bytes=938819584 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939731104 ingress_bytes=1291141120 link_bytes=1291141120 token_bytes=1878550688 weight_movement_bytes=352321536 total_nvlink_bytes=2230872224 weight_to_token_ratio=0.187550 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.023674 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229851323e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=914328 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940438424 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879962520 weight_movement_bytes=528482304 total_nvlink_bytes=2408444824 weight_to_token_ratio=0.281113 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=912328 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940338120 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879763912 weight_movement_bytes=176160768 total_nvlink_bytes=2055924680 weight_to_token_ratio=0.093714 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917592 combine_payload_bytes=939491328 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940408920 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879900248 weight_movement_bytes=176160768 total_nvlink_bytes=2056061016 weight_to_token_ratio=0.093708 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114692 dedup_ratio=0.500008 router_tokens_per_expert=[0, 8191, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 8191, 0, 0, 8192, 0, 7511, 0, 5, 8192, 0, 0, 0, 8192, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 16, 8183, 2, 0, 16, 8192, 0, 6381, 8189, 8192, 14, 0, 0, 0, 0, 0, 8191, 8192, 0, 2328, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16328, 16694, 16382, 15032, 16382, 16434, 12784, 21038] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=109974 dedup_ratio=0.500031 router_tokens_per_expert=[0, 8192, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 8192, 0, 0, 8192, 0, 7829, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 66, 8192, 0, 0, 4, 8191, 0, 7476, 8192, 8192, 17, 0, 0, 0, 0, 0, 8192, 8192, 0, 822, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16252, 13794, 16324, 18722, 16392, 13504, 14986, 21106] combine_rows_per_src=[16384, 16384, 16376, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114558 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 8192, 0, 0, 8192, 0, 5953, 0, 1, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 27, 8186, 0, 0, 13, 8192, 0, 6524, 8192, 8192, 10, 0, 0, 0, 0, 0, 8192, 8192, 0, 3820, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16338, 16514, 16384, 11908, 16384, 16452, 29452, 7640] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917512 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=0 egress_bytes=940425224 ingress_bytes=1027588096 link_bytes=1027588096 token_bytes=1879932936 weight_movement_bytes=88080384 total_nvlink_bytes=1968013320 weight_to_token_ratio=0.046853 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.754064401e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=116412 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 8192, 0, 0, 8192, 0, 7323, 0, 7, 8192, 0, 0, 0, 8192, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 23, 8185, 2, 0, 28, 8192, 0, 6340, 8186, 8192, 6, 0, 0, 0, 0, 0, 8189, 8192, 0, 2517, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16300, 16778, 16384, 14660, 16382, 16476, 12992, 21100] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=898616 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940357176 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879815736 weight_movement_bytes=352321536 total_nvlink_bytes=2232137272 weight_to_token_ratio=0.187423 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916984 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441080 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965176 weight_movement_bytes=352321536 total_nvlink_bytes=2232286712 weight_to_token_ratio=0.187409 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=121006 dedup_ratio=0.500023 router_tokens_per_expert=[7, 7918, 0, 8184, 1, 0, 7, 0, 0, 0, 8, 0, 0, 423, 6, 1, 20, 0, 5, 8185, 8, 8, 8182, 10, 7489, 10, 73, 8170, 0, 0, 6, 8184, 8186, 0, 0, 8166, 0, 1, 0, 39, 13, 1362, 8148, 13, 6, 4, 8184, 3, 4924, 8175, 8178, 12, 2, 107, 8, 0, 9, 8123, 8183, 0, 2315, 6, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[15850, 17200, 16452, 15128, 16406, 19082, 10072, 20888] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114518 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8190, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 8192, 0, 0, 8192, 0, 7558, 0, 20, 8192, 0, 0, 0, 8192, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 82, 8161, 1, 0, 36, 8191, 0, 6575, 8175, 8192, 29, 0, 0, 0, 0, 0, 8187, 8192, 0, 1964, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16230, 16890, 16384, 15156, 16382, 16558, 13174, 20302] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=113482 dedup_ratio=0.500023 router_tokens_per_expert=[0, 8192, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 8192, 0, 0, 8192, 0, 7795, 0, 10, 8192, 1, 0, 0, 8192, 8189, 0, 0, 8192, 0, 0, 0, 0, 0, 188, 8165, 0, 0, 27, 8191, 0, 6662, 8183, 8192, 29, 0, 0, 0, 0, 0, 8189, 8192, 0, 1706, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16378] dup_rows_per_dst=[17596, 16554, 16386, 25714, 16378, 15392, 19646, 3412] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114690 dedup_ratio=0.500008 router_tokens_per_expert=[1, 8122, 0, 8160, 0, 0, 1, 0, 1, 0, 1, 0, 0, 80, 1, 55, 3, 0, 1, 8191, 1, 1, 8185, 2, 7362, 1, 11, 8188, 3, 0, 1, 8190, 8183, 0, 0, 8189, 0, 0, 0, 6, 4, 406, 8176, 1, 1, 9, 8181, 1, 3530, 8187, 8190, 9, 1, 3, 1, 0, 1, 8190, 8191, 1, 5049, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16184, 16648, 16384, 14752, 16374, 17174, 17966, 15592] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=924400 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=352321536 egress_bytes=940448496 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879972592 weight_movement_bytes=704643072 total_nvlink_bytes=2584615664 weight_to_token_ratio=0.374816 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 4, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=942752 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940417696 ingress_bytes=1467957248 link_bytes=1467957248 token_bytes=1879892640 weight_movement_bytes=528482304 total_nvlink_bytes=2408374944 weight_to_token_ratio=0.281124 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219947833e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114690 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 8190, 8192, 8192, 0, 0, 0, 0, 8182, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 61, 8192, 8186, 0, 0, 1, 0, 11, 0, 8192, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 8131, 4, 0, 0, 0, 0, 1, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 16376, 16494, 16408, 16382, 16272, 16384, 16376] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=916808 combine_payload_bytes=939491328 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940408136 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879899464 weight_movement_bytes=528482304 total_nvlink_bytes=2408381768 weight_to_token_ratio=0.281123 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=912656 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940387600 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879862544 weight_movement_bytes=176160768 total_nvlink_bytes=2056023312 weight_to_token_ratio=0.093709 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940425216 ingress_bytes=1203748864 link_bytes=1203748864 token_bytes=1879932928 weight_movement_bytes=264241152 total_nvlink_bytes=2144174080 weight_to_token_ratio=0.140559 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740491169e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130370 payload_rows_remote=113986 dup_rows=131774 dup_rows_remote=115360 dedup_ratio=0.502678 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 159, 8192, 8191, 0, 0, 0, 0, 33, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16334, 16384, 15732, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16206, 16500, 14808, 16450, 17012, 17950, 16434, 16414] combine_rows_per_src=[16334, 16384, 15732, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114714 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 52, 8192, 8190, 0, 0, 0, 0, 13, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16358, 16484, 16410, 16384, 16280, 16384, 16388] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917512 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441608 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965704 weight_movement_bytes=176160768 total_nvlink_bytes=2056126472 weight_to_token_ratio=0.093704 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114622 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 8188, 8192, 8192, 0, 0, 0, 0, 8171, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 134, 8191, 8183, 0, 0, 5, 0, 28, 0, 8192, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 8057, 4, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16376, 16350, 16632, 16450, 16380, 16122, 16384, 16378] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=933773312 dispatch_meta_bytes=917384 combine_payload_bytes=933773312 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=934690696 ingress_bytes=1286094848 link_bytes=1286094848 token_bytes=1868464008 weight_movement_bytes=352321536 total_nvlink_bytes=2220785544 weight_to_token_ratio=0.188562 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.201594306e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917608 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441704 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965800 weight_movement_bytes=352321536 total_nvlink_bytes=2232287336 weight_to_token_ratio=0.187408 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130996 payload_rows_remote=114612 dup_rows=131148 dup_rows_remote=114726 dedup_ratio=0.500290 router_tokens_per_expert=[8177, 0, 0, 0, 8185, 8181, 8117, 0, 2, 9, 0, 8187, 15, 0, 0, 8, 8184, 0, 355, 0, 0, 227, 8184, 8184, 0, 7, 0, 63, 0, 8, 8182, 7, 8158, 1, 7, 8, 12, 7875, 4, 9, 7911, 0, 4, 7, 9, 2, 0, 8185, 10, 8182, 8179, 0, 7, 0, 0, 25, 7, 8185, 0, 0, 1, 0, 2, 0] payload_rows_per_dst=[16370, 16384, 16370, 16380, 16340, 16384, 16384, 16384] dup_rows_per_dst=[16234, 16420, 17530, 16508, 15808, 15852, 16422, 16374] combine_rows_per_src=[16370, 16384, 16370, 16380, 16340, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917240 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441336 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965432 weight_movement_bytes=176160768 total_nvlink_bytes=2056126200 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114954 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 8179, 8192, 8192, 0, 0, 0, 0, 8162, 0, 0, 13, 0, 8192, 0, 0, 0, 0, 136, 8192, 8181, 0, 0, 9, 0, 31, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8056, 2, 0, 0, 0, 0, 1, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 5, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16358, 16350, 16634, 16464, 16384, 16118, 16384, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114220 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 8192, 8192, 8192, 0, 0, 0, 0, 8164, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 93, 8191, 8179, 0, 0, 4, 0, 30, 0, 8191, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 8098, 10, 0, 0, 0, 0, 1, 8192, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16852, 16340, 16608, 15982, 16462, 16218, 16236, 16374] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114684 dup_rows=131078 dup_rows_remote=114528 dedup_ratio=0.500023 router_tokens_per_expert=[8190, 0, 0, 0, 8187, 8191, 8178, 0, 1, 1, 0, 8179, 2, 0, 6, 1, 8191, 1, 62, 0, 0, 32, 8187, 8184, 0, 1, 2, 0, 12, 1, 8187, 1, 8188, 0, 1, 1, 2, 8150, 1, 1, 8119, 4, 1, 1, 1, 0, 5, 8191, 0, 8191, 8190, 0, 1, 30, 1, 1, 1, 8189, 0, 0, 0, 4, 1, 0] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16348, 16378, 16550, 16404, 16306, 16260, 16444, 16388] combine_rows_per_src=[16382, 16384, 16382, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 4, 1, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114688 dup_rows=131076 dup_rows_remote=117626 dedup_ratio=0.500015 router_tokens_per_expert=[3747, 8186, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 8192, 0, 0, 0, 2, 0, 8192, 8190, 0, 8192, 4372, 0, 0, 8192, 0, 8188, 0, 2, 0, 0, 73, 3, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[19160, 16384, 16394, 16388, 13450, 16532, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938901504 dispatch_meta_bytes=917352 combine_payload_bytes=938901504 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939818856 ingress_bytes=1291223040 link_bytes=1291223040 token_bytes=1878720360 weight_movement_bytes=352321536 total_nvlink_bytes=2231041896 weight_to_token_ratio=0.187533 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.230310041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918568 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940442664 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879966760 weight_movement_bytes=176160768 total_nvlink_bytes=2056127528 weight_to_token_ratio=0.093704 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=915632 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=792723456 egress_bytes=940439728 ingress_bytes=1732247552 link_bytes=1732247552 token_bytes=1879963824 weight_movement_bytes=792723456 total_nvlink_bytes=2672687280 weight_to_token_ratio=0.421670 gemm_rows_defined=131328 gemm_rows_charged=135168 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.028409 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699863215e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130896 payload_rows_remote=114512 dup_rows=131248 dup_rows_remote=114658 dedup_ratio=0.500671 router_tokens_per_expert=[4329, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 8192, 8192, 0, 8192, 3792, 0, 2, 8192, 0, 8192, 0, 0, 0, 0, 68, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16208, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8520, 16364, 16486, 16214, 24164, 16520, 16390, 16590] combine_rows_per_src=[16208, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[3670, 8184, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 4465, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 57, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[23708, 16384, 16400, 16384, 8930, 16498, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=916848 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=940408176 ingress_bytes=1732214784 link_bytes=1732214784 token_bytes=1879899504 weight_movement_bytes=792723456 total_nvlink_bytes=2672622960 weight_to_token_ratio=0.421684 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699679728e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114686 dedup_ratio=0.500000 router_tokens_per_expert=[4051, 8186, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8190, 0, 6, 0, 0, 8192, 0, 0, 0, 1, 0, 8192, 8192, 0, 8192, 4075, 0, 0, 8192, 0, 8187, 0, 1, 0, 0, 66, 5, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[24474, 16384, 16392, 16386, 8150, 16518, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=929256 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940453352 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879977448 weight_movement_bytes=704643072 total_nvlink_bytes=2584620520 weight_to_token_ratio=0.374815 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938082304 dispatch_meta_bytes=916680 combine_payload_bytes=938082304 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938998984 ingress_bytes=1114243072 link_bytes=1114243072 token_bytes=1877081288 weight_movement_bytes=176160768 total_nvlink_bytes=2053242056 weight_to_token_ratio=0.093848 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239296095e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917496 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441592 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965688 weight_movement_bytes=176160768 total_nvlink_bytes=2056126456 weight_to_token_ratio=0.093704 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130974 payload_rows_remote=114592 dup_rows=131170 dup_rows_remote=114790 dedup_ratio=0.500374 router_tokens_per_expert=[2025, 8138, 1, 5, 0, 0, 8183, 0, 9, 7, 8, 7, 5, 0, 8160, 8, 8045, 2, 7, 0, 0, 8186, 68, 0, 0, 0, 8184, 10, 0, 0, 64, 0, 8185, 8172, 0, 8184, 6155, 9, 153, 8179, 30, 8185, 0, 0, 0, 14, 5, 4, 6, 8185, 0, 0, 6, 0, 8178, 6, 8, 8, 8, 0, 0, 8079, 8183, 8] payload_rows_per_dst=[16288, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[4050, 16394, 16232, 16500, 28960, 16450, 16380, 16204] combine_rows_per_src=[16288, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114650 dup_rows=131110 dup_rows_remote=114506 dedup_ratio=0.500145 router_tokens_per_expert=[3077, 8168, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 1, 13, 0, 0, 8192, 0, 0, 0, 8, 1, 8192, 8190, 0, 8192, 5003, 0, 0, 8192, 0, 8181, 0, 1, 0, 0, 112, 8, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16346, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[6154, 16384, 16412, 16402, 26386, 16604, 16384, 16384] combine_rows_per_src=[16346, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=107128 dedup_ratio=0.500000 router_tokens_per_expert=[3782, 8181, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8191, 0, 6, 0, 0, 8192, 0, 0, 0, 8, 1, 8192, 8191, 0, 8192, 4269, 0, 0, 8192, 0, 8183, 0, 1, 0, 0, 140, 5, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[23944, 16384, 16380, 16402, 9698, 15496, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114706 dedup_ratio=0.500038 router_tokens_per_expert=[2503, 8180, 1, 1, 0, 0, 8190, 3, 1, 1, 1, 1, 1, 0, 8188, 5, 8166, 0, 1, 0, 0, 8187, 23, 3, 0, 0, 8191, 1, 1, 0, 2, 4, 8191, 8185, 0, 8191, 5658, 1, 17, 8190, 2, 8185, 0, 5, 0, 2, 30, 5, 1, 8191, 0, 1, 1, 0, 8190, 0, 1, 1, 1, 0, 0, 8176, 8191, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16374, 16384, 16384, 16384] dup_rows_per_dst=[21372, 16394, 16376, 16396, 11348, 16454, 16384, 16358] combine_rows_per_src=[16384, 16384, 16384, 16384, 16374, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114688 dup_rows=131082 dup_rows_remote=116034 dedup_ratio=0.500038 router_tokens_per_expert=[0, 8192, 0, 654, 0, 0, 9, 0, 8192, 0, 8192, 13, 0, 0, 0, 0, 8189, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 7525, 3, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16374, 16384, 16384, 16384] dup_rows_per_dst=[17710, 16410, 16378, 16384, 15048, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16374, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=113104 dedup_ratio=0.500084 router_tokens_per_expert=[0, 8192, 0, 1959, 0, 0, 27, 0, 8192, 0, 8192, 1, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 6217, 0, 8170, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16362, 16384, 16384, 16384] dup_rows_per_dst=[5588, 16190, 14974, 16384, 12412, 16404, 31152, 17990] combine_rows_per_src=[16384, 16384, 16384, 16384, 16362, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938737664 dispatch_meta_bytes=917528 combine_payload_bytes=938737664 prefetch_bytes=88080384 reduce_bytes=0 egress_bytes=939655192 ingress_bytes=1026818048 link_bytes=1026818048 token_bytes=1878392856 weight_movement_bytes=88080384 total_nvlink_bytes=1966473240 weight_to_token_ratio=0.046891 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.749752454e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=916624 combine_payload_bytes=939212800 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940129424 ingress_bytes=1115373568 link_bytes=1115373568 token_bytes=1879342224 weight_movement_bytes=176160768 total_nvlink_bytes=2055502992 weight_to_token_ratio=0.093735 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245626401e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114684 dedup_ratio=0.500069 router_tokens_per_expert=[0, 8192, 0, 557, 0, 1, 11, 0, 8192, 0, 8192, 11, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 7621, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16366, 16384, 16384, 16384] dup_rows_per_dst=[17522, 16406, 16384, 16384, 15242, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16366, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917536 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940359712 ingress_bytes=1203683328 link_bytes=1203683328 token_bytes=1879801888 weight_movement_bytes=264241152 total_nvlink_bytes=2144043040 weight_to_token_ratio=0.140569 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740124195e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=887264 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940411360 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879935456 weight_movement_bytes=528482304 total_nvlink_bytes=2408417760 weight_to_token_ratio=0.281117 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.019886 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114704 dedup_ratio=0.500061 router_tokens_per_expert=[0, 8192, 0, 721, 0, 0, 19, 0, 8192, 0, 8192, 26, 0, 0, 0, 0, 8186, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 7444, 4, 8175, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16368, 16384, 16384, 16384] dup_rows_per_dst=[17864, 16436, 16374, 16384, 14878, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16368, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=922888 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940446984 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879971080 weight_movement_bytes=176160768 total_nvlink_bytes=2056131848 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=911080 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940254952 ingress_bytes=1467826176 link_bytes=1467826176 token_bytes=1879598824 weight_movement_bytes=528482304 total_nvlink_bytes=2408081128 weight_to_token_ratio=0.281168 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219213885e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917416 combine_payload_bytes=939376640 prefetch_bytes=88080384 reduce_bytes=176160768 egress_bytes=940294056 ingress_bytes=1203617792 link_bytes=1203617792 token_bytes=1879670696 weight_movement_bytes=264241152 total_nvlink_bytes=2143911848 weight_to_token_ratio=0.140578 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739757220e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917504 combine_payload_bytes=939393024 prefetch_bytes=0 reduce_bytes=88080384 egress_bytes=940310528 ingress_bytes=1027473408 link_bytes=1027473408 token_bytes=1879703552 weight_movement_bytes=88080384 total_nvlink_bytes=1967783936 weight_to_token_ratio=0.046859 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.753422196e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29965433184 prefetch_bytes=3170893824 reduce_bytes=3170893824 weight_movement_bytes=6341787648 egress_bytes=14990047584 ingress_bytes=21317173248 link_bytes=21317173248 total_nvlink_bytes=36307220832 weight_to_token_ratio=0.211637 prefetch_share_of_ingress=0.148748 dedup_ratio=0.501845 prefetch_slots=72 unused_prefetch_slots=440 slot_utilization=0.140625 reduce_slots=36 remote_demand=36 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021188 gemm_inner_pad_upper_bound=33020 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.460454498e-05 prefetch_bytes_per_gemm_flop=1.109730114e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29884658424 prefetch_bytes=2994733056 reduce_bytes=3699376128 weight_movement_bytes=6694109184 egress_bytes=14949544696 ingress_bytes=21629222912 link_bytes=21629222912 total_nvlink_bytes=36578767608 weight_to_token_ratio=0.223998 prefetch_share_of_ingress=0.138458 dedup_ratio=0.502919 prefetch_slots=68 unused_prefetch_slots=444 slot_utilization=0.132812 reduce_slots=42 remote_demand=34 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021307 gemm_inner_pad_upper_bound=32766 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.569663739e-05 prefetch_bytes_per_gemm_flop=1.048078441e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130664 payload_rows_remote=114684 dup_rows=131480 dup_rows_remote=115858 dedup_ratio=0.501556 router_tokens_per_expert=[0, 8185, 6, 721, 1, 265, 0, 0, 8185, 7, 8183, 0, 13, 0, 0, 5, 8183, 0, 544, 8165, 5, 8180, 4, 15, 4, 6, 21, 3, 0, 8172, 0, 8180, 0, 7, 3, 7464, 0, 8178, 0, 0, 9, 0, 8, 0, 0, 35, 13, 8, 4, 9, 7979, 0, 0, 11, 8184, 7599, 0, 8137, 5, 8181, 8166, 0, 12, 7] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 15980, 16384] dup_rows_per_dst=[17942, 16402, 17444, 16392, 14920, 16488, 15622, 16270] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 15980, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=114726 dedup_ratio=0.500107 router_tokens_per_expert=[0, 8192, 0, 1248, 0, 0, 28, 0, 8192, 0, 8190, 22, 0, 0, 0, 0, 8186, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 6920, 8, 8165, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16356, 16384, 16384, 16384] dup_rows_per_dst=[18936, 16424, 16384, 16384, 13830, 16374, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16356, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29707202032 prefetch_bytes=2994733056 reduce_bytes=2113929216 weight_movement_bytes=5108662272 egress_bytes=14860856816 ingress_bytes=19955007488 link_bytes=19955007488 total_nvlink_bytes=34815864304 weight_to_token_ratio=0.171967 prefetch_share_of_ingress=0.150074 dedup_ratio=0.505450 prefetch_slots=68 unused_prefetch_slots=444 slot_utilization=0.132812 reduce_slots=24 remote_demand=34 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021662 gemm_inner_pad_upper_bound=31496 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.983732019e-05 prefetch_bytes_per_gemm_flop=1.048078441e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130992 payload_rows_remote=114608 dup_rows=131152 dup_rows_remote=114566 dedup_ratio=0.500305 router_tokens_per_expert=[0, 8190, 1, 1140, 1, 83, 9, 0, 8190, 1, 8187, 7, 2, 0, 0, 0, 8187, 0, 100, 8187, 1, 8190, 0, 3, 0, 1, 4, 0, 0, 8188, 1, 8191, 0, 1, 1, 7040, 5, 8181, 0, 0, 1, 9, 1, 0, 0, 16, 1, 1, 2, 1, 8156, 0, 0, 2, 8191, 8049, 0, 8183, 0, 8190, 8174, 0, 2, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16374, 16382, 16318, 16382] dup_rows_per_dst=[18846, 16390, 16586, 16386, 14082, 16422, 16102, 16338] combine_rows_per_src=[16384, 16384, 16384, 16384, 16374, 16382, 16318, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114408 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 135, 0, 0, 5, 0, 8191, 0, 8192, 12, 0, 0, 1, 0, 8188, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8044, 5, 8176, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16664, 16414, 16548, 16378, 30764, 16234, 3130, 14940] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29972594552 prefetch_bytes=2906652672 reduce_bytes=3787456512 weight_movement_bytes=6694109184 egress_bytes=14993653624 ingress_bytes=21673050112 link_bytes=21673050112 total_nvlink_bytes=36666703736 weight_to_token_ratio=0.223341 prefetch_share_of_ingress=0.134114 dedup_ratio=0.501909 prefetch_slots=66 unused_prefetch_slots=446 slot_utilization=0.128906 reduce_slots=43 remote_demand=33 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021958 gemm_inner_pad_upper_bound=31242 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.585002116e-05 prefetch_bytes_per_gemm_flop=1.017252604e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=922168 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940413496 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879904824 weight_movement_bytes=528482304 total_nvlink_bytes=2408387128 weight_to_token_ratio=0.281122 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=917544 combine_payload_bytes=939294720 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940212264 ingress_bytes=1291616256 link_bytes=1291616256 token_bytes=1879506984 weight_movement_bytes=352321536 total_nvlink_bytes=2231828520 weight_to_token_ratio=0.187454 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232511886e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938868736 dispatch_meta_bytes=916696 combine_payload_bytes=938868736 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939785432 ingress_bytes=1467351040 link_bytes=1467351040 token_bytes=1878654168 weight_movement_bytes=528482304 total_nvlink_bytes=2407136472 weight_to_token_ratio=0.281309 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.216553322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916384 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940440480 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879964576 weight_movement_bytes=176160768 total_nvlink_bytes=2056125344 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29942468768 prefetch_bytes=2730491904 reduce_bytes=2202009600 weight_movement_bytes=4932501504 egress_bytes=14978519200 ingress_bytes=19896451072 link_bytes=19896451072 total_nvlink_bytes=34874970272 weight_to_token_ratio=0.164733 prefetch_share_of_ingress=0.137235 dedup_ratio=0.502228 prefetch_slots=62 unused_prefetch_slots=450 slot_utilization=0.121094 reduce_slots=25 remote_demand=31 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.022076 gemm_inner_pad_upper_bound=31750 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.963238801e-05 prefetch_bytes_per_gemm_flop=9.556009312e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29959144528 prefetch_bytes=2554331136 reduce_bytes=3435134976 weight_movement_bytes=5989466112 egress_bytes=14986921040 ingress_bytes=20961689600 link_bytes=20961689600 total_nvlink_bytes=35948610640 weight_to_token_ratio=0.199921 prefetch_share_of_ingress=0.121857 dedup_ratio=0.501908 prefetch_slots=58 unused_prefetch_slots=454 slot_utilization=0.113281 reduce_slots=39 remote_demand=29 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021129 gemm_inner_pad_upper_bound=32893 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.336044495e-05 prefetch_bytes_per_gemm_flop=8.939492582e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29868152880 prefetch_bytes=2906652672 reduce_bytes=2290089984 weight_movement_bytes=5196742656 egress_bytes=14941427760 ingress_bytes=20123467776 link_bytes=20123467776 total_nvlink_bytes=35064895536 weight_to_token_ratio=0.173989 prefetch_share_of_ingress=0.144441 dedup_ratio=0.503204 prefetch_slots=66 unused_prefetch_slots=446 slot_utilization=0.128906 reduce_slots=26 remote_demand=33 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021484 gemm_inner_pad_upper_bound=32004 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.042688725e-05 prefetch_bytes_per_gemm_flop=1.017252604e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29688123776 prefetch_bytes=2642411520 reduce_bytes=2202009600 weight_movement_bytes=4844421120 egress_bytes=14851314048 ingress_bytes=19681230848 link_bytes=19681230848 total_nvlink_bytes=34532544896 weight_to_token_ratio=0.163177 prefetch_share_of_ingress=0.134260 dedup_ratio=0.505730 prefetch_slots=60 unused_prefetch_slots=452 slot_utilization=0.117188 reduce_slots=25 remote_demand=30 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021899 gemm_inner_pad_upper_bound=30607 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.887917337e-05 prefetch_bytes_per_gemm_flop=9.247750947e-06 anomalies=none - [2026-09-16 21:39:33] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 610.2 | throughput per GPU (TFLOP/s/GPU): 546.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.273962E+00 | loss scale: 1.0 | grad norm: 5.594 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122330 payload_rows_remote=106384 dup_rows=139814 dup_rows_remote=116234 dedup_ratio=0.533348 router_tokens_per_expert=[6377, 235, 840, 470, 3923, 1, 2844, 384, 1652, 5601, 2032, 703, 2210, 1924, 0, 314, 726, 2861, 2690, 1764, 10, 828, 3154, 2616, 100, 72, 5961, 2347, 5731, 710, 5007, 2413, 66, 3472, 5050, 758, 822, 332, 4815, 1283, 3542, 85, 5182, 3102, 1, 3284, 1483, 1772, 6285, 182, 1853, 10, 90, 1798, 1, 1514, 0, 100, 503, 2765, 5273, 1539, 2018, 5592] payload_rows_per_dst=[15960, 15148, 15946, 14510, 15622, 14464, 15076, 15604] dup_rows_per_dst=[23750, 13724, 23580, 13198, 17574, 14276, 13736, 19976] combine_rows_per_src=[15960, 15148, 15946, 14510, 15622, 14464, 15076, 15604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122918 payload_rows_remote=108128 dup_rows=139226 dup_rows_remote=126386 dedup_ratio=0.531105 router_tokens_per_expert=[5390, 369, 957, 398, 4040, 2, 3267, 301, 960, 5566, 2131, 943, 2511, 2074, 0, 390, 836, 2335, 2820, 1820, 5, 1129, 3003, 2570, 92, 46, 5481, 1699, 5718, 677, 5181, 2514, 53, 3405, 5205, 527, 850, 173, 4876, 1595, 2898, 79, 5008, 2851, 0, 4204, 1587, 1784, 6776, 118, 1845, 0, 94, 2373, 2, 1867, 1, 80, 263, 2754, 5616, 1844, 1971, 5148] payload_rows_per_dst=[15432, 15344, 14986, 14790, 15590, 15194, 15808, 15774] dup_rows_per_dst=[17766, 13806, 16198, 12840, 17778, 19480, 21778, 19580] combine_rows_per_src=[15432, 15344, 14986, 14790, 15590, 15194, 15808, 15774] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122802 payload_rows_remote=107048 dup_rows=139342 dup_rows_remote=118666 dedup_ratio=0.531548 router_tokens_per_expert=[5385, 297, 861, 409, 2815, 0, 3025, 332, 1440, 5516, 2224, 1434, 2524, 2488, 0, 366, 548, 2619, 3147, 1630, 5, 936, 2727, 2536, 101, 55, 5522, 2182, 5668, 764, 5638, 2717, 63, 3549, 5686, 708, 976, 284, 4222, 2064, 3257, 60, 4881, 2829, 0, 4016, 1453, 1512, 6222, 146, 1705, 0, 151, 2994, 1, 1328, 1, 64, 199, 2430, 5983, 1476, 2041, 4860] payload_rows_per_dst=[14404, 15424, 14690, 15756, 15752, 15434, 15754, 15588] dup_rows_per_dst=[11844, 16560, 13606, 18202, 19352, 20582, 20676, 18520] combine_rows_per_src=[14404, 15424, 14690, 15756, 15752, 15434, 15754, 15588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123788 payload_rows_remote=107946 dup_rows=138356 dup_rows_remote=119974 dedup_ratio=0.527786 router_tokens_per_expert=[5981, 224, 521, 490, 3697, 0, 3307, 265, 1277, 6668, 2460, 1210, 1908, 3183, 3, 403, 832, 1991, 3108, 1082, 11, 1388, 2450, 2196, 81, 48, 6025, 1992, 5886, 773, 5332, 2487, 23, 2999, 5149, 638, 854, 231, 4863, 1135, 2129, 48, 4861, 2983, 0, 5074, 1441, 1812, 6621, 93, 2171, 5, 138, 2111, 3, 1273, 1, 83, 256, 2786, 5041, 1477, 1930, 5564] payload_rows_per_dst=[16052, 15842, 15908, 15826, 15360, 14648, 14950, 15202] dup_rows_per_dst=[23506, 18382, 21392, 20014, 16782, 12326, 9880, 16074] combine_rows_per_src=[16052, 15842, 15908, 15826, 15360, 14648, 14950, 15202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122682 payload_rows_remote=107132 dup_rows=139462 dup_rows_remote=121066 dedup_ratio=0.532005 router_tokens_per_expert=[5370, 362, 1096, 488, 4085, 0, 2706, 254, 1366, 6087, 2004, 1307, 2367, 2210, 0, 543, 900, 2218, 2990, 2072, 3, 1111, 2258, 2826, 101, 54, 5502, 2167, 6014, 635, 5183, 3031, 69, 4012, 4834, 265, 1005, 244, 5112, 1086, 2545, 112, 4849, 2781, 2, 4207, 1298, 1539, 6275, 40, 1813, 4, 171, 2207, 18, 2301, 3, 67, 146, 3051, 5860, 1825, 1649, 4372] payload_rows_per_dst=[15036, 15622, 14544, 15694, 15048, 15384, 15804, 15550] dup_rows_per_dst=[13686, 16146, 14212, 17652, 14814, 22674, 21882, 18396] combine_rows_per_src=[15036, 15622, 14544, 15694, 15048, 15384, 15804, 15550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=871497728 dispatch_meta_bytes=890472 combine_payload_bytes=871497728 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=872388200 ingress_bytes=1047658496 link_bytes=1047658496 token_bytes=1743885928 weight_movement_bytes=176160768 total_nvlink_bytes=1920046696 weight_to_token_ratio=0.101016 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=12544 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.866450263e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122700 payload_rows_remote=107562 dup_rows=139444 dup_rows_remote=123816 dedup_ratio=0.531937 router_tokens_per_expert=[5125, 328, 510, 434, 4247, 1, 2938, 322, 1726, 5972, 2255, 928, 1983, 2866, 0, 440, 921, 1932, 3058, 1354, 1, 1366, 3159, 2617, 109, 216, 5788, 2120, 5375, 955, 5433, 2923, 42, 2986, 5600, 373, 768, 281, 4042, 1291, 2130, 67, 5156, 3376, 0, 4425, 1138, 1540, 5848, 187, 1912, 1, 155, 2871, 1, 1362, 0, 61, 335, 3094, 5620, 2460, 1569, 4979] payload_rows_per_dst=[14846, 15492, 14774, 15788, 15138, 15252, 15808, 15602] dup_rows_per_dst=[12964, 16848, 14042, 19300, 15628, 20412, 19616, 20634] combine_rows_per_src=[14846, 15492, 14774, 15788, 15138, 15252, 15808, 15602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 2, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=885784576 dispatch_meta_bytes=938056 combine_payload_bytes=885784576 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=886722632 ingress_bytes=1414266880 link_bytes=1414266880 token_bytes=1772507208 weight_movement_bytes=528482304 total_nvlink_bytes=2300989512 weight_to_token_ratio=0.298155 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=21376 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.919304184e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=876937216 dispatch_meta_bytes=902856 combine_payload_bytes=876937216 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=877840072 ingress_bytes=1053097984 link_bytes=1053097984 token_bytes=1754777288 weight_movement_bytes=176160768 total_nvlink_bytes=1930938056 weight_to_token_ratio=0.100389 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=26112 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.896909126e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=884293632 dispatch_meta_bytes=911680 combine_payload_bytes=884293632 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=885205312 ingress_bytes=1060454400 link_bytes=1060454400 token_bytes=1769498944 weight_movement_bytes=176160768 total_nvlink_bytes=1945659712 weight_to_token_ratio=0.099554 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=24064 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.938101985e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=877625344 dispatch_meta_bytes=912792 combine_payload_bytes=877625344 prefetch_bytes=0 reduce_bytes=616562688 egress_bytes=878538136 ingress_bytes=1494188032 link_bytes=1494188032 token_bytes=1756163480 weight_movement_bytes=616562688 total_nvlink_bytes=2372726168 weight_to_token_ratio=0.351085 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.023674 gemm_active_groups=18 gemm_peak_group_rows=19712 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.366829274e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120746 payload_rows_remote=104752 dup_rows=141398 dup_rows_remote=117514 dedup_ratio=0.539391 router_tokens_per_expert=[5567, 502, 1221, 322, 4341, 3, 2627, 194, 1260, 5123, 2507, 1447, 2965, 1989, 0, 539, 1240, 2465, 3377, 2161, 9, 1023, 2409, 2764, 98, 54, 5470, 1670, 4891, 524, 4971, 2583, 66, 3342, 5163, 300, 1222, 250, 4868, 1375, 2704, 156, 4520, 2954, 12, 3874, 1493, 1423, 6363, 113, 1488, 21, 109, 2226, 18, 1593, 1, 79, 549, 3211, 5834, 1943, 1538, 5948] payload_rows_per_dst=[15994, 15894, 15938, 15446, 15680, 14186, 14626, 12982] dup_rows_per_dst=[23884, 24732, 24614, 20042, 18594, 12750, 9236, 7546] combine_rows_per_src=[15994, 15894, 15938, 15446, 15680, 14186, 14626, 12982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=881147904 dispatch_meta_bytes=925512 combine_payload_bytes=881147904 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=882073416 ingress_bytes=1145389056 link_bytes=1145389056 token_bytes=1763221320 weight_movement_bytes=264241152 total_nvlink_bytes=2027462472 weight_to_token_ratio=0.149863 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=20864 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.413700605e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128336 payload_rows_remote=111952 dup_rows=133808 dup_rows_remote=119116 dedup_ratio=0.510437 router_tokens_per_expert=[0, 1, 0, 0, 8191, 0, 8180, 0, 8108, 0, 1, 1102, 8192, 1, 1, 961, 5, 0, 0, 1, 0, 0, 1020, 6321, 0, 7718, 0, 4106, 0, 8186, 37, 154, 0, 0, 5, 0, 16, 0, 2, 4858, 8191, 0, 1, 0, 0, 0, 0, 8125, 3, 0, 0, 8112, 8190, 0, 0, 0, 6796, 8192, 480, 8142, 0, 0, 0, 7673] payload_rows_per_dst=[16384, 16382, 16384, 16382, 16384, 16384, 16384, 13652] dup_rows_per_dst=[16360, 20276, 14692, 23996, 25108, 16250, 16226, 900] combine_rows_per_src=[16384, 16382, 16384, 16382, 16384, 16384, 16384, 13652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122800 payload_rows_remote=107664 dup_rows=139344 dup_rows_remote=119398 dedup_ratio=0.531555 router_tokens_per_expert=[5357, 313, 530, 441, 3818, 0, 2947, 104, 1164, 6365, 2139, 1128, 1754, 3453, 0, 587, 836, 2501, 3342, 1511, 7, 1141, 2215, 2304, 72, 230, 4772, 1877, 6176, 1102, 5576, 2914, 37, 2979, 5332, 462, 1132, 270, 4832, 1672, 2310, 63, 4680, 3501, 3, 4024, 1447, 1513, 6261, 147, 1790, 1, 23, 2859, 0, 1500, 6, 67, 219, 2663, 6046, 1682, 2149, 4726] payload_rows_per_dst=[14564, 15596, 14602, 15590, 15640, 15136, 15988, 15684] dup_rows_per_dst=[12456, 17584, 13112, 17496, 17792, 19946, 21526, 19432] combine_rows_per_src=[14564, 15596, 14602, 15590, 15640, 15136, 15988, 15684] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129624 payload_rows_remote=113240 dup_rows=132520 dup_rows_remote=109856 dedup_ratio=0.505524 router_tokens_per_expert=[1, 0, 0, 3, 8188, 1, 8187, 2, 8056, 2, 0, 2230, 8188, 3, 2, 1736, 7, 3, 2, 0, 1, 1, 197, 6374, 1, 7728, 5, 3427, 0, 8138, 71, 152, 1, 0, 11, 3, 4, 0, 0, 4812, 8185, 4, 1, 3, 1, 1, 0, 7941, 1, 0, 3, 7990, 8188, 3, 1, 2, 6113, 8189, 1049, 8166, 3, 1, 2, 7688] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16382, 16384, 14940] dup_rows_per_dst=[16380, 24008, 13158, 22664, 14942, 15890, 15992, 9486] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16382, 16384, 14940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=858128384 dispatch_meta_bytes=889064 combine_payload_bytes=858128384 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=859017448 ingress_bytes=1034289152 link_bytes=1034289152 token_bytes=1717145832 weight_movement_bytes=176160768 total_nvlink_bytes=1893306600 weight_to_token_ratio=0.102589 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=23552 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.791587518e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127548 payload_rows_remote=111164 dup_rows=134596 dup_rows_remote=118246 dedup_ratio=0.513443 router_tokens_per_expert=[0, 0, 0, 0, 8188, 1, 8188, 0, 8170, 0, 0, 2522, 8191, 0, 0, 728, 15, 0, 2, 1, 0, 0, 829, 6183, 1, 7086, 0, 3881, 0, 8189, 19, 67, 1, 1, 1, 0, 4, 1, 0, 4393, 8192, 0, 0, 0, 1, 0, 0, 8119, 2, 0, 0, 8175, 8186, 0, 4, 0, 6790, 8190, 571, 8145, 0, 1, 0, 8034] payload_rows_per_dst=[16382, 14048, 16384, 15200, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16372, 6736, 17838, 6908, 8800, 16240, 16350, 45352] combine_rows_per_src=[16382, 14048, 16384, 15200, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125782 payload_rows_remote=109398 dup_rows=136362 dup_rows_remote=110322 dedup_ratio=0.520180 router_tokens_per_expert=[0, 1, 0, 0, 8192, 3, 8163, 5, 8135, 4, 1, 2421, 8192, 0, 3, 2475, 7, 0, 0, 1, 0, 0, 183, 7101, 1, 6952, 6, 2878, 12, 8171, 132, 81, 0, 3, 0, 0, 4, 2, 2, 5453, 8188, 4, 0, 0, 6, 0, 0, 8186, 2, 0, 0, 7947, 8188, 0, 14, 1, 5418, 8189, 779, 8152, 0, 2, 0, 7412] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16382, 11100] dup_rows_per_dst=[16344, 26040, 16540, 18088, 25746, 16384, 15922, 1298] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16382, 11100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 2, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121422 payload_rows_remote=105038 dup_rows=140722 dup_rows_remote=94700 dedup_ratio=0.536812 router_tokens_per_expert=[1, 0, 0, 0, 8191, 1, 8179, 0, 8016, 3, 0, 1304, 8186, 0, 1, 1195, 27, 0, 0, 0, 0, 0, 92, 6658, 0, 7648, 1, 3609, 13, 8192, 496, 371, 0, 0, 0, 0, 1, 2, 6, 5617, 8192, 8, 0, 0, 0, 0, 1, 7756, 0, 0, 0, 7902, 8191, 0, 9, 0, 6728, 8192, 602, 7907, 0, 0, 0, 7774] payload_rows_per_dst=[16384, 8306, 16382, 14814, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16360, 2834, 21684, 6782, 11252, 17840, 17948, 46022] combine_rows_per_src=[16384, 8306, 16382, 14814, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=917110784 dispatch_meta_bytes=924272 combine_payload_bytes=917110784 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=918035056 ingress_bytes=1269432320 link_bytes=1269432320 token_bytes=1835145840 weight_movement_bytes=352321536 total_nvlink_bytes=2187467376 weight_to_token_ratio=0.191986 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.108291105e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 2, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=927662080 dispatch_meta_bytes=892384 combine_payload_bytes=927662080 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=928554464 ingress_bytes=1544224768 link_bytes=1544224768 token_bytes=1856216544 weight_movement_bytes=616562688 total_nvlink_bytes=2472779232 weight_to_token_ratio=0.332161 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=29568 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.647014110e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=121272 dedup_ratio=0.500038 router_tokens_per_expert=[0, 1, 0, 0, 8192, 2, 8179, 1, 8131, 6, 0, 2312, 8190, 0, 1, 1620, 23, 0, 0, 0, 0, 0, 174, 6467, 0, 7694, 2, 3384, 7, 8185, 46, 667, 0, 1, 5, 0, 27, 1, 0, 4875, 8189, 8, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 8143, 8192, 0, 1, 0, 5362, 8188, 510, 8167, 0, 0, 0, 8029] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] dup_rows_per_dst=[16366, 24136, 13318, 23546, 9810, 16190, 16288, 11428] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=896188416 dispatch_meta_bytes=878880 combine_payload_bytes=896188416 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=897067296 ingress_bytes=1512751104 link_bytes=1512751104 token_bytes=1793255712 weight_movement_bytes=616562688 total_nvlink_bytes=2409818400 weight_to_token_ratio=0.343823 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=31488 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.470774729e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=881983488 dispatch_meta_bytes=908248 combine_payload_bytes=881983488 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=882891736 ingress_bytes=1234305024 link_bytes=1234305024 token_bytes=1764875224 weight_movement_bytes=352321536 total_nvlink_bytes=2117196760 weight_to_token_ratio=0.199630 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=17280 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.911592910e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=910655488 dispatch_meta_bytes=917640 combine_payload_bytes=910655488 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=911573128 ingress_bytes=1086816256 link_bytes=1086816256 token_bytes=1822228616 weight_movement_bytes=176160768 total_nvlink_bytes=1998389384 weight_to_token_ratio=0.096673 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.085717374e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=860471296 dispatch_meta_bytes=798952 combine_payload_bytes=860471296 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=861270248 ingress_bytes=1565114368 link_bytes=1565114368 token_bytes=1721741544 weight_movement_bytes=704643072 total_nvlink_bytes=2426384616 weight_to_token_ratio=0.409262 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.763987150e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=943800 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940385976 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879828152 weight_movement_bytes=352321536 total_nvlink_bytes=2232149688 weight_to_token_ratio=0.187422 gemm_rows_defined=132992 gemm_rows_charged=135168 gemm_tail_pad_rows=2176 gemm_inner_pad_upper_bound=2794 gemm_tail_pad_fraction=0.016098 gemm_active_groups=22 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128190 payload_rows_remote=111806 dup_rows=133954 dup_rows_remote=117428 dedup_ratio=0.510994 router_tokens_per_expert=[2, 0, 0, 0, 8187, 13, 8187, 0, 8023, 0, 0, 654, 8190, 0, 5, 476, 9, 4, 0, 6, 3, 0, 195, 6812, 0, 8019, 0, 5911, 1, 8172, 35, 472, 2, 0, 1, 5, 0, 0, 2, 4608, 8187, 24, 0, 0, 1, 0, 3, 7998, 0, 0, 2, 8105, 8186, 3, 8, 5, 6729, 8183, 414, 7953, 5, 0, 4, 7268] payload_rows_per_dst=[16384, 16384, 16384, 16374, 16384, 16380, 16384, 13516] dup_rows_per_dst=[16526, 17966, 22070, 20570, 23754, 16046, 16234, 788] combine_rows_per_src=[16384, 16384, 16384, 16374, 16384, 16380, 16384, 13516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114680 dup_rows=131082 dup_rows_remote=114720 dedup_ratio=0.500038 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 8153, 0, 0, 0, 8189, 2, 8187, 0, 0, 8192, 0, 0, 8191, 0, 0, 8181, 0, 0, 4, 8063, 8190, 0, 8177, 0, 0, 1, 0, 8192, 1, 2, 106, 8188, 8191, 0, 0, 8192, 0, 0, 0, 12, 0, 81, 0, 8191, 0, 0, 0, 1, 0, 8191, 0, 0, 8192, 0, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16378, 16382, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16308, 16378, 16362, 16110, 16592, 16564, 16382, 16386] combine_rows_per_src=[16384, 16378, 16382, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114850 dedup_ratio=0.500015 router_tokens_per_expert=[3, 1, 0, 0, 0, 1, 8165, 1, 0, 2, 8189, 43, 8185, 0, 3, 8183, 0, 0, 8189, 3, 3, 8185, 0, 17, 12, 8101, 8181, 0, 8188, 2, 2, 0, 1, 8187, 3, 0, 41, 8188, 8188, 3, 3, 8169, 4, 0, 3, 15, 0, 15, 3, 8188, 4, 3, 2, 0, 3, 8188, 1, 3, 8189, 2, 0, 2, 2, 3] payload_rows_per_dst=[16382, 16384, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16334, 16460, 16410, 16226, 16464, 16398, 16398, 16386] combine_rows_per_src=[16382, 16384, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114698 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 13, 0, 5, 8192, 0, 0, 0, 8173, 14, 8192, 0, 0, 8192, 3, 0, 8190, 0, 0, 8168, 6, 0, 1, 8179, 8168, 0, 8181, 0, 0, 0, 0, 8192, 0, 6, 21, 8172, 8184, 0, 0, 8192, 0, 0, 0, 42, 0, 8, 0, 8192, 0, 0, 0, 9, 0, 8188, 0, 0, 8189, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16420, 16374, 16350, 16306, 16382, 16468, 16394, 16378] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 4, 0, 0, 8048, 0, 0, 0, 8178, 4, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 8177, 1, 0, 7, 8172, 8183, 0, 8175, 0, 0, 0, 0, 8192, 0, 5, 12, 8171, 8191, 0, 0, 8176, 0, 0, 0, 41, 0, 181, 0, 8192, 0, 0, 0, 4, 0, 8192, 0, 0, 8189, 0, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16104, 16218, 16356, 16278, 16574, 16770, 16392, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917600 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940376160 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834720 weight_movement_bytes=176160768 total_nvlink_bytes=2055995488 weight_to_token_ratio=0.093711 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114702 dedup_ratio=0.500061 router_tokens_per_expert=[0, 0, 0, 1, 0, 1, 8192, 0, 0, 0, 8182, 34, 8189, 0, 0, 8192, 0, 0, 8190, 0, 0, 8180, 2, 3, 19, 8142, 8183, 0, 8177, 0, 0, 0, 0, 8190, 0, 3, 39, 8190, 8179, 0, 0, 8191, 0, 0, 0, 7, 0, 8, 0, 8191, 0, 0, 0, 3, 0, 8191, 0, 0, 8191, 0, 1, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16380, 16372, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16426, 16370, 16304, 16438, 16394, 16386, 16386] combine_rows_per_src=[16384, 16384, 16380, 16372, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=915914752 dispatch_meta_bytes=916936 combine_payload_bytes=915914752 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=916831688 ingress_bytes=1092075520 link_bytes=1092075520 token_bytes=1832746440 weight_movement_bytes=176160768 total_nvlink_bytes=2008907208 weight_to_token_ratio=0.096118 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.115167057e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130220 payload_rows_remote=113836 dup_rows=131924 dup_rows_remote=115686 dedup_ratio=0.503250 router_tokens_per_expert=[0, 0, 0, 0, 8192, 1, 8182, 0, 8124, 0, 0, 1350, 8191, 0, 0, 3195, 20, 0, 0, 0, 0, 0, 143, 5735, 0, 7543, 0, 2151, 14, 8181, 64, 105, 0, 0, 1, 0, 5, 0, 1, 5824, 8192, 1, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 8058, 8191, 0, 0, 0, 6571, 8192, 586, 8135, 0, 0, 0, 8006] payload_rows_per_dst=[16384, 16384, 16380, 15548, 16384, 16384, 16382, 16374] dup_rows_per_dst=[16366, 19492, 11780, 12124, 11662, 16238, 16116, 28146] combine_rows_per_src=[16384, 16384, 16380, 15548, 16384, 16384, 16382, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=918144 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940425856 ingress_bytes=1467990016 link_bytes=1467990016 token_bytes=1879933568 weight_movement_bytes=528482304 total_nvlink_bytes=2408415872 weight_to_token_ratio=0.281118 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220131320e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114560 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 0, 11, 0, 0, 8187, 0, 0, 0, 8187, 13, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 8187, 1, 0, 3, 8121, 8174, 0, 8180, 0, 0, 0, 0, 8192, 2, 5, 66, 8191, 8190, 0, 0, 8192, 0, 0, 0, 10, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16372, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16396, 16400, 16376, 16224, 16524, 16398, 16384, 16382] combine_rows_per_src=[16384, 16384, 16384, 16372, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917544 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441640 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965736 weight_movement_bytes=528482304 total_nvlink_bytes=2408448040 weight_to_token_ratio=0.281113 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441616 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965712 weight_movement_bytes=176160768 total_nvlink_bytes=2056126480 weight_to_token_ratio=0.093704 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917496 combine_payload_bytes=939393024 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940310520 ingress_bytes=1291714560 link_bytes=1291714560 token_bytes=1879703544 weight_movement_bytes=352321536 total_nvlink_bytes=2232025080 weight_to_token_ratio=0.187435 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233062348e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=932544512 dispatch_meta_bytes=918088 combine_payload_bytes=932544512 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=933462600 ingress_bytes=1108705280 link_bytes=1108705280 token_bytes=1866007112 weight_movement_bytes=176160768 total_nvlink_bytes=2042167880 weight_to_token_ratio=0.094405 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.208286772e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=916944 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940342736 ingress_bytes=1467908096 link_bytes=1467908096 token_bytes=1879768528 weight_movement_bytes=528482304 total_nvlink_bytes=2408250832 weight_to_token_ratio=0.281142 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219672603e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=108688 dedup_ratio=0.500023 router_tokens_per_expert=[0, 0, 0, 0, 8185, 0, 0, 8186, 8184, 8077, 0, 0, 0, 8185, 0, 8192, 8190, 0, 6856, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 8191, 7983, 27, 6, 0, 3, 20, 0, 1, 0, 0, 0, 8192, 8192, 0, 0, 1962, 8, 0, 4, 0, 8191, 7788, 66, 0, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16378, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16358, 16150, 22390, 16378, 16032, 16412, 11650, 15708] combine_rows_per_src=[16384, 16378, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 2, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114720 dedup_ratio=0.500061 router_tokens_per_expert=[3, 4, 0, 5, 8152, 3, 0, 8187, 8186, 8119, 0, 3, 0, 8162, 4, 8183, 8188, 0, 6897, 8188, 4, 4, 6, 0, 0, 0, 0, 1, 3, 3, 3, 8183, 1, 3, 10, 8187, 7824, 24, 3, 0, 5, 133, 3, 6, 1, 0, 0, 8181, 8188, 0, 3, 1979, 12, 0, 4, 0, 8178, 7781, 49, 2, 0, 0, 1, 3] payload_rows_per_dst=[16384, 16372, 16384, 16384, 16384, 16384, 16382, 16382] dup_rows_per_dst=[16324, 16244, 13814, 16368, 15720, 16606, 20366, 15646] combine_rows_per_src=[16384, 16372, 16384, 16384, 16384, 16384, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114644 dup_rows=131120 dup_rows_remote=114374 dedup_ratio=0.500183 router_tokens_per_expert=[5, 0, 6, 13, 0, 0, 7988, 0, 0, 0, 8177, 10, 8181, 0, 5, 8184, 0, 5, 8185, 0, 4, 8167, 0, 33, 6, 8124, 8177, 4, 8168, 0, 0, 0, 8, 8182, 6, 0, 39, 8173, 8183, 4, 5, 8176, 9, 0, 4, 39, 0, 190, 5, 8187, 5, 5, 0, 11, 3, 8187, 5, 5, 8186, 7, 2, 4, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16378, 16366, 16372, 16376, 16384] dup_rows_per_dst=[16746, 16406, 16482, 17344, 15722, 15670, 16360, 16390] combine_rows_per_src=[16380, 16384, 16384, 16378, 16366, 16372, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=893480 combine_payload_bytes=939474944 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=940368424 ingress_bytes=1820278784 link_bytes=1820278784 token_bytes=1879843368 weight_movement_bytes=880803840 total_nvlink_bytes=2760647208 weight_to_token_ratio=0.468552 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=28160 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019280137e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114688 dup_rows=131080 dup_rows_remote=114784 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8171, 8182, 8148, 0, 0, 0, 8192, 0, 8192, 8187, 0, 7184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 3, 0, 0, 8191, 7982, 64, 27, 0, 7, 149, 0, 17, 0, 0, 0, 8192, 8192, 0, 0, 1127, 41, 0, 12, 0, 8192, 7995, 85, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16376, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16342, 16296, 30742, 16312, 16150, 16718, 2360, 16160] combine_rows_per_src=[16384, 16376, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127632 payload_rows_remote=111248 dup_rows=134512 dup_rows_remote=109300 dedup_ratio=0.513123 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8182, 8170, 8020, 0, 0, 0, 8192, 0, 8192, 8189, 0, 6480, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 8192, 7761, 52, 26, 0, 2, 45, 0, 2, 0, 0, 0, 8192, 8192, 2, 0, 4640, 48, 0, 6, 0, 8192, 5652, 71, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16364, 12964, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16364, 15722, 84, 16604, 18292, 16458, 25776, 25212] combine_rows_per_src=[16384, 16364, 12964, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130514 payload_rows_remote=114130 dup_rows=131630 dup_rows_remote=110858 dedup_ratio=0.502129 router_tokens_per_expert=[0, 0, 0, 0, 8191, 0, 0, 8177, 8185, 8164, 0, 0, 0, 8190, 0, 8192, 8182, 0, 6898, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 1, 0, 0, 8191, 7753, 13, 7, 0, 3, 36, 0, 4, 0, 0, 0, 8192, 8192, 1, 0, 2168, 20, 0, 5, 0, 8192, 7694, 58, 0, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16374, 15836, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16352, 16328, 11548, 16340, 18322, 16462, 20772, 15506] combine_rows_per_src=[16384, 16374, 15836, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917568 combine_payload_bytes=939393024 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940310592 ingress_bytes=1291714560 link_bytes=1291714560 token_bytes=1879703616 weight_movement_bytes=352321536 total_nvlink_bytes=2232025152 weight_to_token_ratio=0.187435 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233062348e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=916072 combine_payload_bytes=939163648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940079720 ingress_bytes=1291485184 link_bytes=1291485184 token_bytes=1879243368 weight_movement_bytes=352321536 total_nvlink_bytes=2231564904 weight_to_token_ratio=0.187481 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231777938e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 4, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=115216 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 8188, 0, 0, 8175, 8187, 8154, 0, 0, 0, 8187, 0, 8192, 8192, 0, 7166, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 8192, 7876, 47, 5, 0, 5, 72, 0, 26, 0, 0, 0, 8192, 8192, 0, 0, 1943, 10, 0, 12, 0, 8189, 7426, 71, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16342, 16308, 14332, 16362, 15856, 16570, 20314, 14988] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114652 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 8191, 0, 0, 0, 8192, 16, 8192, 0, 0, 8192, 0, 0, 8191, 0, 0, 8186, 0, 1, 13, 8159, 8187, 0, 8179, 0, 0, 0, 0, 8192, 0, 0, 27, 8172, 8190, 0, 0, 8183, 0, 0, 0, 19, 0, 12, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16386, 16416, 16372, 16316, 16394, 16424, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=934952960 dispatch_meta_bytes=899952 combine_payload_bytes=934952960 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=935852912 ingress_bytes=1111113728 link_bytes=1111113728 token_bytes=1870805872 weight_movement_bytes=176160768 total_nvlink_bytes=2046966640 weight_to_token_ratio=0.094163 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.221773075e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917888 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=940441984 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879966080 weight_movement_bytes=704643072 total_nvlink_bytes=2584609152 weight_to_token_ratio=0.374817 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.023674 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=911343616 dispatch_meta_bytes=882192 combine_payload_bytes=911343616 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=912225808 ingress_bytes=1087504384 link_bytes=1087504384 token_bytes=1823569424 weight_movement_bytes=176160768 total_nvlink_bytes=1999730192 weight_to_token_ratio=0.096602 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.089570604e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=919616 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940443712 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879967808 weight_movement_bytes=176160768 total_nvlink_bytes=2056128576 weight_to_token_ratio=0.093704 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=114638 dedup_ratio=0.500061 router_tokens_per_expert=[0, 8192, 1, 0, 153, 0, 0, 0, 8184, 8177, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 8192, 0, 0, 7, 1, 8191, 0, 0, 14, 25, 8192, 0, 86, 8192, 0, 0, 8192, 8188, 8191, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 8033, 0, 8057, 0, 8192, 8187, 0, 0, 9, 0] payload_rows_per_dst=[16384, 16368, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16692, 16354, 16450, 16398, 16634, 16370, 16068, 16122] combine_rows_per_src=[16384, 16368, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917352 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425064 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932776 weight_movement_bytes=176160768 total_nvlink_bytes=2056093544 weight_to_token_ratio=0.093706 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114668 dup_rows=131098 dup_rows_remote=114694 dedup_ratio=0.500099 router_tokens_per_expert=[2, 8189, 1, 0, 71, 0, 3, 4, 8181, 8161, 8187, 3, 3, 4, 0, 2, 0, 0, 3, 16, 0, 3, 85, 0, 8188, 3, 0, 5, 1, 8187, 1, 6, 7, 21, 8176, 0, 204, 8188, 0, 0, 8185, 8188, 8182, 3, 0, 3, 4, 0, 0, 8188, 3, 0, 0, 0, 7958, 1, 8064, 3, 8187, 8186, 3, 3, 3, 3] payload_rows_per_dst=[16384, 16372, 16384, 16378, 16378, 16384, 16384, 16382] dup_rows_per_dst=[16530, 16336, 16564, 16404, 16814, 16386, 15916, 16148] combine_rows_per_src=[16384, 16372, 16384, 16378, 16378, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114376 dedup_ratio=0.500008 router_tokens_per_expert=[5, 6, 0, 4, 8157, 6, 1, 8179, 8164, 7950, 0, 5, 0, 8149, 0, 8176, 8184, 0, 6910, 8184, 6, 6, 8, 0, 0, 0, 0, 0, 4, 3, 4, 8177, 0, 6, 15, 8186, 7863, 66, 51, 1, 7, 80, 5, 23, 0, 0, 2, 8181, 8186, 0, 5, 3935, 26, 2, 18, 0, 8180, 5874, 56, 11, 0, 0, 0, 5] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16698, 17734, 29530, 16344, 15992, 14946, 7962, 11868] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917240 combine_payload_bytes=939393024 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940310264 ingress_bytes=1291714560 link_bytes=1291714560 token_bytes=1879703288 weight_movement_bytes=352321536 total_nvlink_bytes=2232024824 weight_to_token_ratio=0.187435 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233062348e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=115664 dedup_ratio=0.500099 router_tokens_per_expert=[0, 8192, 1, 0, 401, 0, 0, 0, 8179, 8155, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 107, 0, 8192, 0, 1, 3, 0, 8189, 0, 0, 7, 19, 8192, 0, 112, 8191, 0, 0, 8192, 8186, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 7716, 0, 8074, 0, 8191, 8186, 0, 0, 7, 0] payload_rows_per_dst=[16384, 16358, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17186, 16310, 16602, 16386, 16658, 16372, 15434, 16150] combine_rows_per_src=[16384, 16358, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=111362 dedup_ratio=0.500107 router_tokens_per_expert=[0, 8192, 0, 0, 104, 0, 0, 0, 8173, 8168, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 61, 0, 8192, 0, 0, 10, 0, 8185, 0, 0, 11, 82, 8192, 0, 197, 8192, 0, 0, 8192, 8185, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7831, 0, 8136, 0, 8191, 8179, 0, 0, 22, 0] payload_rows_per_dst=[16384, 16356, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[13286, 14430, 14754, 16364, 12874, 18124, 21530, 19738] combine_rows_per_src=[16384, 16356, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131032 payload_rows_remote=114688 dup_rows=131112 dup_rows_remote=114842 dedup_ratio=0.500153 router_tokens_per_expert=[0, 8192, 0, 0, 387, 0, 0, 0, 8171, 8136, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 8, 0, 8185, 0, 0, 14, 61, 8192, 0, 70, 8192, 0, 0, 8192, 8182, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7832, 0, 8108, 0, 8192, 8166, 0, 0, 20, 0] payload_rows_per_dst=[16384, 16344, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17158, 16270, 16392, 16386, 16674, 16364, 15664, 16204] combine_rows_per_src=[16384, 16344, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=917448 combine_payload_bytes=939360256 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940277704 ingress_bytes=1115521024 link_bytes=1115521024 token_bytes=1879637960 weight_movement_bytes=176160768 total_nvlink_bytes=2055798728 weight_to_token_ratio=0.093721 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246452093e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114668 dup_rows=131092 dup_rows_remote=114536 dedup_ratio=0.500076 router_tokens_per_expert=[0, 8192, 1, 0, 171, 0, 0, 0, 8182, 8170, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8192, 0, 0, 6, 0, 8187, 0, 0, 5, 55, 8192, 0, 26, 8192, 0, 0, 8192, 8187, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8070, 0, 8078, 0, 8192, 8184, 0, 0, 12, 0] payload_rows_per_dst=[16384, 16364, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16728, 16340, 16418, 16386, 16556, 16360, 16140, 16164] combine_rows_per_src=[16384, 16364, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=916248 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940423960 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879931672 weight_movement_bytes=176160768 total_nvlink_bytes=2056092440 weight_to_token_ratio=0.093706 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=921304 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940232408 ingress_bytes=1115471872 link_bytes=1115471872 token_bytes=1879543512 weight_movement_bytes=176160768 total_nvlink_bytes=2055704280 weight_to_token_ratio=0.093725 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246176862e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=904088 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940198808 ingress_bytes=1467777024 link_bytes=1467777024 token_bytes=1879493528 weight_movement_bytes=528482304 total_nvlink_bytes=2407975832 weight_to_token_ratio=0.281183 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218938654e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114182 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8146, 8189, 8174, 0, 0, 0, 8190, 0, 8192, 8187, 0, 6305, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8192, 7337, 13, 4, 0, 1, 253, 0, 8, 0, 0, 0, 8192, 8192, 0, 0, 2724, 22, 0, 0, 0, 8191, 7970, 24, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16292, 16346, 12610, 16364, 14708, 16900, 21876, 15986] combine_rows_per_src=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918120 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940442216 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879966312 weight_movement_bytes=528482304 total_nvlink_bytes=2408448616 weight_to_token_ratio=0.281113 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=916816 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940277072 ingress_bytes=1291681792 link_bytes=1291681792 token_bytes=1879637328 weight_movement_bytes=352321536 total_nvlink_bytes=2231958864 weight_to_token_ratio=0.187441 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232878861e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114970 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 2, 46, 8191, 0, 0, 0, 8051, 0, 1, 0, 8192, 0, 8192, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 1, 8192, 8190, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 115, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 13, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16422, 16102, 16374, 16382, 16384, 16612, 16412] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=915440 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940357616 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879799792 weight_movement_bytes=176160768 total_nvlink_bytes=2055960560 weight_to_token_ratio=0.093713 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114700 dedup_ratio=0.500038 router_tokens_per_expert=[0, 3, 8188, 0, 1, 3, 0, 1, 1, 0, 0, 4, 8005, 0, 0, 236, 8184, 4, 2, 0, 7975, 1, 0, 2, 8188, 0, 8188, 0, 4, 2, 3, 8186, 8187, 1, 0, 3, 0, 8187, 8183, 8188, 4, 3, 5, 8188, 8188, 4, 8188, 3, 5, 3, 0, 150, 3, 2, 8187, 4, 0, 4, 5, 8187, 1, 7, 0, 1] payload_rows_per_dst=[16382, 16384, 16378, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16386, 16484, 15958, 16382, 16366, 16406, 16698, 16402] combine_rows_per_src=[16382, 16384, 16378, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130892 payload_rows_remote=114508 dup_rows=131252 dup_rows_remote=114808 dedup_ratio=0.500687 router_tokens_per_expert=[4, 8186, 0, 1, 25, 0, 6, 5, 8156, 8177, 8179, 5, 6, 6, 0, 0, 0, 1, 0, 10, 0, 5, 525, 4, 8186, 12, 0, 8, 0, 8181, 0, 0, 25, 52, 8173, 0, 352, 8187, 1, 6, 8181, 8177, 8182, 6, 0, 5, 3, 0, 4, 8185, 5, 0, 0, 0, 7354, 5, 8081, 5, 8187, 8173, 2, 5, 23, 5] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16364, 16384, 16226] dup_rows_per_dst=[16444, 18828, 17432, 16366, 18362, 14734, 14722, 14364] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16364, 16384, 16226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918632 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940442728 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879966824 weight_movement_bytes=176160768 total_nvlink_bytes=2056127592 weight_to_token_ratio=0.093704 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114574 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 1, 193, 8190, 0, 0, 0, 7975, 0, 2, 0, 8192, 0, 8192, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 8192, 8179, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 17, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 56, 0, 2] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16322, 14302, 15952, 18772, 16424, 16384, 16418, 16500] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114554 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 254, 8192, 0, 0, 0, 7932, 0, 2, 0, 8192, 0, 8192, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 0, 8192, 8186, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 69, 0, 0, 8190, 0, 0, 0, 0, 8192, 1, 7, 0, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16764, 15868, 16374, 16372, 16384, 16518, 16404] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917512 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940359688 ingress_bytes=1467924480 link_bytes=1467924480 token_bytes=1879801864 weight_movement_bytes=528482304 total_nvlink_bytes=2408284168 weight_to_token_ratio=0.281137 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219764346e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113882 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 1, 443, 8192, 0, 0, 0, 7732, 0, 7, 0, 8192, 0, 8192, 0, 0, 0, 0, 8177, 8192, 0, 0, 0, 0, 8192, 8170, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 27, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 58, 0, 4] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16394, 17190, 15478, 16354, 16340, 16384, 16424, 16508] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938049536 dispatch_meta_bytes=917264 combine_payload_bytes=938049536 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938966800 ingress_bytes=1114210304 link_bytes=1114210304 token_bytes=1877016336 weight_movement_bytes=176160768 total_nvlink_bytes=2053177104 weight_to_token_ratio=0.093851 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239112608e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114694 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 4, 194, 8187, 0, 0, 0, 7907, 0, 5, 0, 8192, 0, 8192, 0, 0, 0, 0, 8186, 8191, 0, 0, 0, 1, 8192, 8188, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 130, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 10, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16394, 16686, 15814, 16372, 16378, 16384, 16640, 16404] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917040 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940424752 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932464 weight_movement_bytes=176160768 total_nvlink_bytes=2056093232 weight_to_token_ratio=0.093706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916968 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441064 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965160 weight_movement_bytes=176160768 total_nvlink_bytes=2056125928 weight_to_token_ratio=0.093704 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114694 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8192, 0, 0, 332, 0, 0, 0, 8189, 8176, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 8192, 0, 0, 3, 0, 8192, 0, 0, 0, 3, 8192, 0, 58, 8192, 0, 0, 8192, 8191, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7799, 0, 8165, 0, 8192, 8192, 0, 0, 6, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[17048, 16350, 16460, 16390, 16506, 16382, 15598, 16342] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=914280 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940438376 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879962472 weight_movement_bytes=176160768 total_nvlink_bytes=2056123240 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917528 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940441624 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965720 weight_movement_bytes=704643072 total_nvlink_bytes=2584608792 weight_to_token_ratio=0.374817 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 4840, 8190, 0, 8192, 8191, 0, 0, 2, 0, 8187, 0, 0, 8190, 9, 0, 8192, 0, 0, 0, 10, 0, 0, 8192, 0, 0, 0, 0, 9, 0, 0, 8188, 0, 0, 0, 8191, 1, 16, 0, 0, 3321, 0, 8192, 8192, 0, 0, 0, 8192, 8188, 0, 0, 0, 0, 0, 1, 2, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[26064, 16382, 16388, 16404, 16394, 6674, 16376, 16390] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114666 dedup_ratio=0.500015 router_tokens_per_expert=[4, 0, 19, 4, 8188, 2, 0, 2916, 8175, 8, 8185, 8186, 1, 0, 1, 4, 8183, 0, 4, 8086, 4, 0, 8186, 0, 1, 3, 14, 0, 2, 8186, 4, 0, 3, 3, 12, 5, 4, 8132, 0, 0, 2, 8188, 0, 106, 1, 3, 5210, 4, 8187, 8188, 63, 3, 0, 8180, 8177, 1, 0, 4, 3, 4, 1, 35, 0, 8187] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16382] dup_rows_per_dst=[22254, 16366, 16170, 16410, 16310, 10644, 16466, 16456] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917512 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940408840 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879900168 weight_movement_bytes=528482304 total_nvlink_bytes=2408382472 weight_to_token_ratio=0.281123 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114650 dedup_ratio=0.500000 router_tokens_per_expert=[2, 5, 8185, 21, 0, 5, 0, 0, 3, 0, 1, 6, 8138, 4, 0, 54, 8185, 6, 1, 0, 8005, 0, 0, 9, 8186, 7, 8185, 2, 6, 2, 3, 8173, 8185, 3, 1, 5, 1, 8181, 8182, 8185, 7, 0, 0, 8184, 8184, 8, 8186, 27, 8, 5, 0, 10, 0, 2, 8185, 6, 0, 6, 19, 8184, 0, 114, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16422, 16400, 20064, 14064, 16438, 16362, 15286, 16036] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441584 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965680 weight_movement_bytes=528482304 total_nvlink_bytes=2408447984 weight_to_token_ratio=0.281113 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114634 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 1867, 8175, 0, 8192, 8188, 0, 0, 2, 0, 8165, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 12, 0, 0, 8192, 0, 0, 0, 0, 23, 0, 0, 8177, 0, 0, 0, 8192, 4, 59, 0, 0, 6260, 0, 8192, 8191, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 29, 0, 8192] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[3734, 15934, 16800, 16282, 16398, 28976, 16510, 16442] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917400 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940408728 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879900056 weight_movement_bytes=176160768 total_nvlink_bytes=2056060824 weight_to_token_ratio=0.093708 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 1, 3051, 8184, 0, 8192, 8189, 0, 0, 3, 0, 8183, 0, 0, 8190, 3, 0, 8192, 0, 0, 0, 7, 0, 0, 8192, 0, 0, 0, 0, 10, 0, 0, 8174, 0, 0, 0, 8192, 4, 62, 0, 0, 5085, 0, 8192, 8192, 1, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 8, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[6104, 16368, 16386, 16398, 16368, 26668, 16380, 16400] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114688 dup_rows=131086 dup_rows_remote=114740 dedup_ratio=0.500053 router_tokens_per_expert=[0, 0, 0, 0, 8190, 0, 5, 4309, 8175, 0, 8192, 8183, 0, 0, 0, 0, 8153, 0, 0, 8187, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 8192, 0, 0, 0, 0, 31, 0, 0, 8181, 0, 0, 0, 8192, 7, 47, 0, 0, 3841, 0, 8192, 8192, 0, 0, 0, 8192, 8180, 0, 0, 0, 0, 1, 3, 34, 0, 8192] payload_rows_per_dst=[16384, 16370, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[25008, 16346, 16296, 16402, 16424, 7790, 16360, 16460] combine_rows_per_src=[16384, 16370, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917352 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441448 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965544 weight_movement_bytes=176160768 total_nvlink_bytes=2056126312 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114664 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 2683, 8184, 0, 8192, 8190, 0, 0, 1, 0, 8184, 0, 0, 8178, 15, 0, 8192, 0, 0, 0, 10, 0, 0, 8192, 0, 0, 0, 0, 19, 0, 0, 8186, 0, 0, 0, 8192, 0, 48, 0, 0, 5413, 0, 8192, 8192, 0, 0, 0, 8192, 8188, 0, 0, 0, 0, 0, 2, 43, 0, 8192] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[21124, 16368, 16440, 16404, 16410, 11478, 16376, 16474] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917272 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940408600 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879899928 weight_movement_bytes=176160768 total_nvlink_bytes=2056060696 weight_to_token_ratio=0.093708 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=940441616 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965712 weight_movement_bytes=704643072 total_nvlink_bytes=2584608784 weight_to_token_ratio=0.374817 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 264, 8192, 0, 0, 0, 7922, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 8192, 8187, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 46, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 11, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16824, 15844, 16380, 16374, 16384, 16476, 16406] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917712 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441808 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965904 weight_movement_bytes=528482304 total_nvlink_bytes=2408448208 weight_to_token_ratio=0.281113 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917400 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940425112 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932824 weight_movement_bytes=352321536 total_nvlink_bytes=2232254360 weight_to_token_ratio=0.187412 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=105610 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 8192, 99, 0, 0, 8192, 0, 8187, 0, 0, 0, 12, 0, 6812, 8192, 0, 0, 8181, 0, 0, 4551, 8191, 8192, 425, 4593, 8116, 0, 8192, 8189, 0, 8172, 0, 5, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16582, 13638, 25464, 9884, 16356, 16382, 16384, 16384] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=119790 dedup_ratio=0.500008 router_tokens_per_expert=[4, 0, 2, 8188, 23, 4, 0, 8187, 5, 8186, 4, 4, 0, 6, 4, 4780, 8187, 0, 2, 8185, 3, 0, 5948, 8186, 8174, 241, 5634, 7954, 4, 8187, 8185, 1, 8166, 2, 6, 1, 4, 0, 0, 0, 0, 0, 2, 0, 4, 3, 1, 0, 8188, 3, 4, 0, 2, 4, 0, 3, 0, 8186, 8188, 2, 2, 11, 0, 2] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16432, 9594, 28266, 11284, 16348, 16386, 16366, 16398] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131006 payload_rows_remote=114622 dup_rows=131138 dup_rows_remote=110108 dedup_ratio=0.500252 router_tokens_per_expert=[8, 0, 29, 6, 8181, 5, 0, 2295, 8171, 8, 8181, 8186, 8, 1, 0, 6, 8169, 0, 6, 7831, 0, 1, 8183, 0, 0, 3, 7, 0, 5, 8185, 6, 0, 5, 4, 15, 8, 6, 8079, 1, 0, 5, 8185, 0, 29, 2, 0, 6159, 6, 8180, 8184, 127, 5, 0, 8171, 8178, 4, 0, 6, 6, 3, 0, 41, 0, 8182] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16322, 16380] dup_rows_per_dst=[21030, 16958, 15630, 16388, 16410, 12388, 16286, 16048] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16322, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 6, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=881192 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940405288 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879929384 weight_movement_bytes=880803840 total_nvlink_bytes=2760733224 weight_to_token_ratio=0.468530 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=25728 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114644 dup_rows=131116 dup_rows_remote=114728 dedup_ratio=0.500168 router_tokens_per_expert=[0, 0, 0, 8192, 378, 0, 0, 8192, 0, 8183, 0, 0, 0, 43, 0, 6527, 8192, 0, 0, 8170, 0, 0, 4060, 8192, 8192, 475, 5173, 7990, 0, 8192, 8187, 0, 8133, 0, 21, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 2, 0, 0] payload_rows_per_dst=[16384, 16384, 16340, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15576, 13122, 9398, 27174, 16596, 16444, 16418, 16388] combine_rows_per_src=[16384, 16384, 16340, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=937904 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=880803840 egress_bytes=940445616 ingress_bytes=1996472320 link_bytes=1996472320 token_bytes=1879953328 weight_movement_bytes=1056964608 total_nvlink_bytes=2936917936 weight_to_token_ratio=0.562229 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.117941162e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114650 dup_rows=131110 dup_rows_remote=114726 dedup_ratio=0.500145 router_tokens_per_expert=[0, 0, 0, 8192, 34, 0, 0, 8192, 0, 8185, 0, 0, 0, 16, 0, 6910, 8192, 0, 0, 8173, 0, 0, 3812, 8192, 8192, 376, 5477, 7971, 0, 8192, 8186, 0, 8181, 0, 19, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16346, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16452, 13838, 7624, 27648, 16408, 16372, 16384, 16384] combine_rows_per_src=[16384, 16384, 16346, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=104418 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 70, 0, 0, 8192, 0, 8181, 0, 0, 0, 48, 0, 7050, 8192, 0, 0, 8152, 0, 0, 4038, 8192, 8192, 167, 5392, 7914, 0, 8192, 8179, 0, 8116, 0, 26, 0, 0, 0, 0, 3, 0, 0, 8, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16524, 26654, 11900, 10562, 16290, 16394, 16364, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938983424 dispatch_meta_bytes=898920 combine_payload_bytes=938983424 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939882344 ingress_bytes=1115144192 link_bytes=1115144192 token_bytes=1878865768 weight_movement_bytes=176160768 total_nvlink_bytes=2055026536 weight_to_token_ratio=0.093759 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244341991e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114726 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 57, 0, 0, 8192, 0, 8186, 0, 0, 0, 33, 0, 7122, 8192, 0, 0, 8184, 0, 0, 2066, 8192, 8192, 306, 6900, 8125, 0, 8192, 8191, 0, 8164, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16498, 14298, 20500, 14278, 16346, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=917488 combine_payload_bytes=939163648 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940081136 ingress_bytes=1115324416 link_bytes=1115324416 token_bytes=1879244784 weight_movement_bytes=176160768 total_nvlink_bytes=2055405552 weight_to_token_ratio=0.093740 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245351170e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=917504 combine_payload_bytes=939212800 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940130304 ingress_bytes=1291534336 link_bytes=1291534336 token_bytes=1879343104 weight_movement_bytes=352321536 total_nvlink_bytes=2231664640 weight_to_token_ratio=0.187471 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232053169e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=876424 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940400520 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879924616 weight_movement_bytes=176160768 total_nvlink_bytes=2056085384 weight_to_token_ratio=0.093706 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917656 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441752 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965848 weight_movement_bytes=352321536 total_nvlink_bytes=2232287384 weight_to_token_ratio=0.187408 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=104620 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 8189, 0, 0, 1584, 8186, 0, 8192, 8191, 0, 0, 0, 0, 8189, 0, 0, 8190, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 8174, 0, 0, 0, 8192, 0, 22, 0, 0, 6602, 0, 8192, 8192, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 12, 0, 8192] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[5754, 16372, 16962, 16384, 16358, 26454, 16382, 16408] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114662 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 6, 0, 9, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 1, 8192, 1814, 0, 0, 8192, 6089, 62, 188, 0, 0, 35, 0, 8184, 0, 8192, 8191, 0, 0, 0, 8191, 0, 0, 8190, 1, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16384, 16410, 16378, 20014, 12748, 16368, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114654 dup_rows=131106 dup_rows_remote=113622 dedup_ratio=0.500130 router_tokens_per_expert=[1, 0, 0, 8181, 3, 0, 2, 18, 10, 0, 7521, 8176, 0, 4, 4, 0, 12, 4, 7, 5, 0, 1, 0, 3, 1, 8182, 4, 4, 2, 541, 8186, 14, 3, 8188, 1, 0, 8187, 653, 3, 4, 8187, 7253, 305, 149, 0, 5, 378, 3, 8180, 4, 8157, 8184, 4, 0, 0, 8187, 9, 4, 8188, 5, 0, 7758, 0, 8187] payload_rows_per_dst=[16382, 16360, 16384, 16384, 16384, 16384, 16376, 16384] dup_rows_per_dst=[16402, 15070, 16422, 17484, 17694, 16176, 16314, 15544] combine_rows_per_src=[16382, 16360, 16384, 16384, 16384, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114484 dedup_ratio=0.500031 router_tokens_per_expert=[5, 0, 0, 8184, 82, 6, 0, 8184, 7, 8174, 6, 6, 1, 23, 6, 5507, 8184, 0, 6, 8166, 5, 0, 6353, 8185, 8153, 504, 4423, 7707, 6, 8185, 8180, 0, 8127, 38, 23, 0, 7, 0, 0, 0, 0, 6, 4, 0, 6, 5, 0, 3, 8182, 0, 7, 0, 0, 7, 0, 5, 0, 8183, 8186, 3, 3, 29, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16596, 27408, 12714, 8900, 16374, 16328, 16390, 16370] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=877224 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940384936 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879892648 weight_movement_bytes=176160768 total_nvlink_bytes=2056053416 weight_to_token_ratio=0.093708 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917400 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441496 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965592 weight_movement_bytes=352321536 total_nvlink_bytes=2232287128 weight_to_token_ratio=0.187409 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129742 payload_rows_remote=113358 dup_rows=132402 dup_rows_remote=114116 dedup_ratio=0.505074 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 8177, 8192, 0, 0, 0, 0, 23, 0, 12, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 1, 8192, 1160, 0, 0, 8192, 6776, 52, 126, 0, 1, 84, 0, 8174, 0, 8192, 8192, 0, 1, 0, 8191, 0, 0, 8189, 7, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16150, 15288, 16384, 16384, 16384] dup_rows_per_dst=[16350, 16354, 16096, 15540, 6560, 26434, 16782, 18286] combine_rows_per_src=[16384, 16384, 16384, 16150, 15288, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=913104 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940158672 ingress_bytes=1379647488 link_bytes=1379647488 token_bytes=1879404240 weight_movement_bytes=440401920 total_nvlink_bytes=2319806160 weight_to_token_ratio=0.234331 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725450040e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 17, 0, 15, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 298, 0, 0, 8192, 7864, 14, 4, 0, 1, 12, 0, 8170, 0, 8192, 8191, 0, 0, 0, 8188, 0, 0, 8186, 12, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16394, 16384, 16438, 16350, 16980, 15790, 16340, 16396] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114700 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 415, 0, 0, 8192, 7657, 28, 73, 0, 7, 14, 0, 8186, 0, 8192, 8188, 0, 0, 0, 8192, 0, 0, 8189, 2, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 16384, 16412, 16362, 17214, 15558, 16372, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=916656 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940375216 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879833776 weight_movement_bytes=352321536 total_nvlink_bytes=2232155312 weight_to_token_ratio=0.187422 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=112258 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 10, 0, 6, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 1215, 0, 0, 8192, 6628, 43, 283, 0, 7, 21, 0, 8185, 0, 8192, 8186, 0, 0, 0, 8192, 0, 0, 8191, 1, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 16384, 16404, 16366, 18814, 13964, 16370, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=928628736 dispatch_meta_bytes=909896 combine_payload_bytes=928628736 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=929538632 ingress_bytes=1280950272 link_bytes=1280950272 token_bytes=1858167368 weight_movement_bytes=352321536 total_nvlink_bytes=2210488904 weight_to_token_ratio=0.189607 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.172786828e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 4, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 2, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114690 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 14, 0, 0, 8192, 0, 8191, 0, 0, 0, 17, 0, 6244, 8192, 0, 0, 8189, 0, 0, 4099, 8192, 8192, 73, 6115, 8035, 0, 8192, 8190, 0, 8173, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16412, 12520, 15292, 21346, 16354, 16382, 16382, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114690 dedup_ratio=0.500008 router_tokens_per_expert=[8192, 0, 8189, 8192, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 8175, 0, 6919, 0, 0, 0, 25, 0, 7453, 0, 0, 7665, 0, 0, 0, 8186, 0, 0, 8, 8189, 0, 4, 0, 2521, 0, 0, 8191, 7, 8192, 1, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16378, 16388, 16384, 16386, 13854, 30226, 5060, 16398] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917552 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940441648 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965744 weight_movement_bytes=528482304 total_nvlink_bytes=2408448048 weight_to_token_ratio=0.281113 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=907784 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940431880 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879955976 weight_movement_bytes=616562688 total_nvlink_bytes=2496518664 weight_to_token_ratio=0.327967 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=12 unused_prefetch_slots=20 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=6 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 12, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114710 dedup_ratio=0.500031 router_tokens_per_expert=[8188, 0, 8183, 8183, 11, 3, 5, 4, 0, 8187, 2, 4, 0, 8, 9, 8169, 8183, 2, 4, 4, 5, 2, 4, 42, 0, 3, 4, 0, 3, 1, 8178, 0, 8153, 3, 7299, 16, 4, 0, 660, 0, 5197, 0, 4, 7910, 0, 4, 0, 8183, 0, 6, 6, 8184, 0, 8, 1, 3470, 0, 3, 8186, 8, 8188, 1, 0, 8187] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16380, 16380] dup_rows_per_dst=[16394, 16374, 16484, 16370, 15886, 13222, 19960, 16390] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16380, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131026 payload_rows_remote=114652 dup_rows=131118 dup_rows_remote=114726 dedup_ratio=0.500175 router_tokens_per_expert=[0, 0, 0, 8173, 1, 0, 0, 26, 45, 0, 7218, 8167, 0, 7, 7, 0, 32, 7, 13, 9, 0, 7, 4, 5, 30, 8170, 6, 6, 5, 906, 8183, 27, 5, 8184, 0, 0, 8180, 851, 4, 7, 8185, 6896, 508, 208, 0, 0, 300, 5, 8169, 7, 8100, 8186, 6, 0, 0, 8186, 14, 6, 8185, 0, 0, 7643, 0, 8183] payload_rows_per_dst=[16374, 16380, 16384, 16384, 16384, 16382, 16354, 16384] dup_rows_per_dst=[16392, 17760, 16514, 16456, 16682, 15822, 16180, 15312] combine_rows_per_src=[16374, 16380, 16384, 16384, 16384, 16382, 16354, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917512 combine_payload_bytes=939524096 prefetch_bytes=528482304 reduce_bytes=0 egress_bytes=940441608 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965704 weight_movement_bytes=528482304 total_nvlink_bytes=2408448008 weight_to_token_ratio=0.281113 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=2.959280303e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 8] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425216 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932928 weight_movement_bytes=176160768 total_nvlink_bytes=2056093696 weight_to_token_ratio=0.093706 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114650 dup_rows=131110 dup_rows_remote=114568 dedup_ratio=0.500145 router_tokens_per_expert=[8192, 5, 8184, 8192, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 8164, 0, 6880, 0, 0, 0, 115, 0, 6644, 0, 0, 7876, 0, 0, 1, 8177, 0, 0, 23, 8183, 0, 16, 0, 3047, 0, 0, 8192, 27, 8192, 1, 0, 8192] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16372, 16364, 16384, 16384] dup_rows_per_dst=[16046, 16386, 16384, 16390, 13946, 12908, 22508, 16542] combine_rows_per_src=[16378, 16384, 16384, 16384, 16372, 16364, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917560 combine_payload_bytes=939458560 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940376120 ingress_bytes=1291780096 link_bytes=1291780096 token_bytes=1879834680 weight_movement_bytes=352321536 total_nvlink_bytes=2232156216 weight_to_token_ratio=0.187422 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233429322e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=917512 combine_payload_bytes=939229184 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940146696 ingress_bytes=1115389952 link_bytes=1115389952 token_bytes=1879375880 weight_movement_bytes=176160768 total_nvlink_bytes=2055536648 weight_to_token_ratio=0.093734 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245718145e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114696 dedup_ratio=0.500031 router_tokens_per_expert=[8192, 8, 8179, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8192, 0, 8160, 0, 6264, 0, 0, 0, 30, 0, 5884, 0, 0, 7491, 0, 0, 0, 8181, 0, 0, 22, 8172, 0, 12, 0, 4894, 0, 0, 8192, 43, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16378, 16384, 16384] dup_rows_per_dst=[16374, 16384, 16384, 16392, 12526, 26734, 9816, 16470] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 4, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114676 dup_rows=131088 dup_rows_remote=117000 dedup_ratio=0.500061 router_tokens_per_expert=[8192, 4, 8188, 8192, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8178, 0, 7038, 0, 0, 0, 18, 0, 6840, 0, 0, 7429, 0, 0, 0, 8185, 0, 0, 15, 8186, 0, 7, 0, 3233, 0, 0, 8192, 22, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16372, 16384, 16384] dup_rows_per_dst=[16384, 16386, 16384, 16384, 14088, 12166, 22868, 16428] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=110114 dedup_ratio=0.500099 router_tokens_per_expert=[8192, 0, 8183, 8192, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 8176, 0, 7170, 0, 0, 0, 27, 0, 7052, 0, 0, 8031, 0, 0, 0, 8185, 0, 0, 15, 8186, 0, 6, 0, 2292, 0, 0, 8192, 17, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16358, 16384, 16384] dup_rows_per_dst=[16366, 16388, 16384, 16388, 14362, 13808, 20984, 16418] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16358, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=916872 combine_payload_bytes=939212800 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=940129672 ingress_bytes=1643855872 link_bytes=1643855872 token_bytes=1879342472 weight_movement_bytes=704643072 total_nvlink_bytes=2583985544 weight_to_token_ratio=0.374941 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204906704e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917504 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940376064 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834624 weight_movement_bytes=176160768 total_nvlink_bytes=2055995392 weight_to_token_ratio=0.093711 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=116490 dedup_ratio=0.500000 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 913, 0, 0, 8192, 7195, 26, 29, 0, 30, 11, 0, 8192, 0, 8192, 8188, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16380, 16376, 16370, 18210, 14582, 16384, 16386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114686 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8191, 8187, 195, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 1, 0, 8192, 8192, 0, 8168, 2520, 0, 8192, 8192, 8191, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 4, 0, 114, 8192, 8192, 0, 0, 0, 4563, 0, 0, 0, 0, 0, 1225, 0, 0, 0, 7793] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16374, 16774, 16386, 21378, 16382, 16618, 9126, 18034] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=926704 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940352496 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879778288 weight_movement_bytes=176160768 total_nvlink_bytes=2055939056 weight_to_token_ratio=0.093714 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=899104 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940210208 ingress_bytes=1115471872 link_bytes=1115471872 token_bytes=1879521312 weight_movement_bytes=176160768 total_nvlink_bytes=2055682080 weight_to_token_ratio=0.093726 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246176862e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114682 dup_rows=131080 dup_rows_remote=113642 dedup_ratio=0.500031 router_tokens_per_expert=[8116, 0, 40, 4, 4, 0, 8187, 8181, 1085, 5, 3, 0, 0, 0, 4, 4, 0, 0, 0, 10, 4, 55, 3, 8188, 1, 3, 8170, 8186, 0, 8175, 525, 19, 8187, 8188, 8153, 0, 0, 1, 0, 4, 5, 8187, 8178, 20, 0, 2, 0, 54, 8185, 8187, 0, 5, 2, 6642, 0, 0, 0, 5, 5, 137, 5, 4, 2, 7942] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16382, 16384, 16380, 16384] dup_rows_per_dst=[16306, 18530, 16512, 17438, 16308, 16508, 13288, 16190] combine_rows_per_src=[16384, 16384, 16384, 16382, 16382, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131008 payload_rows_remote=114624 dup_rows=131136 dup_rows_remote=114384 dedup_ratio=0.500244 router_tokens_per_expert=[8185, 6, 8186, 8180, 7, 7, 13, 6, 0, 8185, 0, 7, 0, 35, 10, 8135, 8177, 1, 7, 7, 8, 1, 6, 65, 0, 5, 7, 0, 5, 0, 8163, 0, 8133, 5, 7844, 43, 7, 0, 1678, 0, 3771, 0, 7, 7935, 0, 6, 0, 8176, 0, 15, 10, 8173, 0, 16, 5, 3241, 0, 5, 8183, 38, 8184, 0, 0, 8183] payload_rows_per_dst=[16384, 16384, 16380, 16382, 16384, 16384, 16384, 16326] dup_rows_per_dst=[16752, 16382, 16288, 16380, 32534, 10180, 6536, 16084] combine_rows_per_src=[16384, 16384, 16380, 16382, 16384, 16384, 16384, 16326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917496 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441592 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965688 weight_movement_bytes=176160768 total_nvlink_bytes=2056126456 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=924712 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940448808 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879972904 weight_movement_bytes=176160768 total_nvlink_bytes=2056133672 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.022727 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=913296 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940388240 ingress_bytes=1467957248 link_bytes=1467957248 token_bytes=1879863184 weight_movement_bytes=528482304 total_nvlink_bytes=2408345488 weight_to_token_ratio=0.281128 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219947833e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114668 dup_rows=131092 dup_rows_remote=114618 dedup_ratio=0.500076 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8189, 8173, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 8192, 8192, 0, 8149, 1432, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8187, 0, 0, 5, 0, 287, 8192, 8192, 0, 0, 0, 6402, 0, 0, 0, 0, 0, 281, 0, 0, 0, 7959] payload_rows_per_dst=[16384, 16384, 16384, 16364, 16384, 16384, 16384, 16384] dup_rows_per_dst=[5890, 9130, 16038, 10228, 16594, 17094, 39644, 16474] combine_rows_per_src=[16384, 16384, 16384, 16364, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114152 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8185, 8173, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 8192, 8192, 0, 8175, 1729, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 0, 9, 0, 250, 8192, 8192, 0, 0, 0, 5061, 0, 0, 0, 0, 0, 1066, 0, 0, 0, 8044] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16346, 16920, 16384, 19820, 16384, 16890, 10122, 18206] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938999808 dispatch_meta_bytes=916032 combine_payload_bytes=938999808 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=939915840 ingress_bytes=1467482112 link_bytes=1467482112 token_bytes=1878915648 weight_movement_bytes=528482304 total_nvlink_bytes=2407397952 weight_to_token_ratio=0.281270 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217287270e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 8179, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8184, 1309, 0, 8192, 8192, 8190, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 5, 0, 456, 8192, 8192, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 430, 0, 0, 0, 7853] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16358, 16758, 16384, 18986, 16380, 17304, 12336, 16566] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=108556 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8189, 8180, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 8192, 8192, 0, 8176, 620, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 4, 0, 489, 8192, 8192, 0, 0, 0, 5814, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 7884] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16360, 17104, 16382, 6708, 16384, 17368, 22516, 18250] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=917144 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940277400 ingress_bytes=1115521024 link_bytes=1115521024 token_bytes=1879637656 weight_movement_bytes=176160768 total_nvlink_bytes=2055798424 weight_to_token_ratio=0.093721 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.018939 gemm_active_groups=17 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246452093e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=915360 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940439456 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879963552 weight_movement_bytes=352321536 total_nvlink_bytes=2232285088 weight_to_token_ratio=0.187409 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114850 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 1, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 0, 8191, 37, 8111, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 1, 8192, 8192, 0, 1, 3, 2, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 16530, 16222, 16384, 16384, 16388, 16394, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940441616 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965712 weight_movement_bytes=352321536 total_nvlink_bytes=2232287248 weight_to_token_ratio=0.187408 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=892976 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940417072 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879941168 weight_movement_bytes=352321536 total_nvlink_bytes=2232262704 weight_to_token_ratio=0.187411 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114660 dup_rows=131102 dup_rows_remote=115910 dedup_ratio=0.500114 router_tokens_per_expert=[0, 8173, 0, 52, 4, 7881, 3, 3, 0, 85, 2, 1, 0, 0, 8187, 877, 7830, 8150, 8176, 0, 0, 0, 277, 0, 5, 8187, 7587, 0, 4, 7, 8184, 8186, 0, 5, 0, 6, 3, 0, 1, 0, 4, 8187, 8168, 1, 8185, 8108, 4, 0, 111, 0, 0, 6, 3, 0, 16, 0, 1, 3, 8188, 8, 8167, 0, 33, 3] payload_rows_per_dst=[16384, 16378, 16378, 16382, 16384, 16368, 16384, 16384] dup_rows_per_dst=[15848, 18278, 16136, 15192, 16390, 16202, 16634, 16422] combine_rows_per_src=[16384, 16378, 16378, 16382, 16384, 16368, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114688 dup_rows=131090 dup_rows_remote=119178 dedup_ratio=0.500069 router_tokens_per_expert=[8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 6937, 0, 0, 0, 10, 0, 6369, 0, 0, 7770, 0, 0, 0, 8188, 0, 0, 4, 8190, 0, 3, 0, 3479, 0, 0, 8192, 10, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16366, 16384, 16384] dup_rows_per_dst=[16384, 16384, 16384, 16384, 13894, 11912, 23344, 16404] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16366, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 2, 0, 2, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130680 payload_rows_remote=114296 dup_rows=131464 dup_rows_remote=104408 dedup_ratio=0.501495 router_tokens_per_expert=[8071, 0, 83, 6, 7, 0, 8185, 8173, 1218, 8, 5, 0, 0, 0, 6, 6, 0, 0, 0, 11, 7, 101, 4, 8185, 0, 5, 8155, 8179, 0, 8172, 1088, 41, 8182, 8185, 8133, 0, 0, 0, 0, 7, 8, 8183, 8177, 31, 0, 11, 0, 411, 8178, 8184, 0, 34, 4, 5480, 2, 0, 0, 8, 8, 134, 9, 7, 0, 7980] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16378, 16384, 16382, 16002] dup_rows_per_dst=[27056, 18770, 16602, 18598, 16266, 12396, 11014, 10762] combine_rows_per_src=[16384, 16384, 16384, 16382, 16378, 16384, 16382, 16002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918152 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940442248 ingress_bytes=1820327936 link_bytes=1820327936 token_bytes=1879966344 weight_movement_bytes=880803840 total_nvlink_bytes=2760770184 weight_to_token_ratio=0.468521 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019307660e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114682 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 60, 0, 0, 0, 0, 8189, 4, 8126, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 1, 8192, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8191, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 11742, 17954, 18846, 16358, 17012, 16386, 16390] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=922280 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940217000 ingress_bytes=1467777024 link_bytes=1467777024 token_bytes=1879511720 weight_movement_bytes=528482304 total_nvlink_bytes=2407994024 weight_to_token_ratio=0.281181 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218938654e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=935464 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940459560 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879983656 weight_movement_bytes=528482304 total_nvlink_bytes=2408465960 weight_to_token_ratio=0.281110 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=28416 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114624 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 35, 0, 0, 0, 0, 8189, 0, 8151, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 3, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8189, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16448, 16302, 16384, 16384, 16388, 16384, 16398] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=936312832 dispatch_meta_bytes=874816 combine_payload_bytes=936312832 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=937187648 ingress_bytes=1464795136 link_bytes=1464795136 token_bytes=1873500480 weight_movement_bytes=528482304 total_nvlink_bytes=2401982784 weight_to_token_ratio=0.282083 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.202241326e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 32, 0, 0, 0, 0, 8191, 8, 8150, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8189, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16462, 16300, 16384, 16384, 16380, 16386, 16392] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114686 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 0, 8189, 27, 8126, 8192, 8192, 0, 0, 0, 1, 0, 0, 8192, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 1, 8191, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16506, 16254, 16384, 16382, 16384, 16386, 16392] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917480 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940441576 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965672 weight_movement_bytes=176160768 total_nvlink_bytes=2056126440 weight_to_token_ratio=0.093704 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917248 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441344 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965440 weight_movement_bytes=176160768 total_nvlink_bytes=2056126208 weight_to_token_ratio=0.093704 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.025568 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114736 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8188, 0, 8, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8169, 0, 0, 8191, 0, 91, 0, 3, 8192, 0, 0, 0, 8191, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8189, 8182, 0, 28, 8192, 0, 6, 8190, 8192, 2, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16392, 16202, 16336, 16570, 16380, 16414, 16396, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917496 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441592 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965688 weight_movement_bytes=352321536 total_nvlink_bytes=2232287224 weight_to_token_ratio=0.187408 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114004 dedup_ratio=0.500046 router_tokens_per_expert=[4, 7849, 0, 758, 3, 1, 5, 0, 7351, 0, 5, 0, 0, 222, 4, 3, 14, 0, 4, 8180, 4, 4, 8139, 8, 354, 5, 3, 8171, 8, 0, 4, 8187, 8186, 0, 0, 8173, 0, 2, 0, 48, 21, 8187, 8166, 8001, 5, 32, 8181, 0, 5, 8186, 8179, 2, 9, 24, 4, 0, 5, 8176, 8187, 1, 1, 1, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16380, 16378, 16384, 16384] dup_rows_per_dst=[17216, 15160, 16322, 17080, 16438, 16076, 16434, 16358] combine_rows_per_src=[16382, 16384, 16384, 16384, 16380, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114178 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 8191, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 8173, 797, 0, 8192, 8192, 8191, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 255, 8192, 8192, 0, 0, 0, 6516, 0, 0, 0, 0, 0, 438, 0, 0, 0, 8048] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 17086, 16384, 17940, 16382, 16894, 13032, 16972] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130994 payload_rows_remote=114610 dup_rows=131150 dup_rows_remote=114232 dedup_ratio=0.500298 router_tokens_per_expert=[0, 8159, 0, 71, 7, 7675, 5, 4, 0, 174, 0, 1, 0, 0, 8186, 1009, 8091, 8138, 8167, 0, 0, 0, 425, 0, 7, 8184, 7176, 0, 7, 13, 8178, 8183, 0, 8, 0, 13, 5, 0, 0, 0, 6, 8185, 8154, 0, 8184, 7994, 7, 6, 198, 8, 0, 11, 0, 6, 30, 0, 6, 5, 8185, 11, 8124, 0, 60, 6] payload_rows_per_dst=[16384, 16374, 16364, 16384, 16384, 16336, 16384, 16384] dup_rows_per_dst=[16918, 18700, 15584, 15116, 16406, 15998, 16118, 16310] combine_rows_per_src=[16384, 16374, 16364, 16384, 16384, 16336, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917696 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441792 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965888 weight_movement_bytes=176160768 total_nvlink_bytes=2056126656 weight_to_token_ratio=0.093704 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=914720 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940340512 ingress_bytes=1467908096 link_bytes=1467908096 token_bytes=1879766304 weight_movement_bytes=528482304 total_nvlink_bytes=2408248608 weight_to_token_ratio=0.281143 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219672603e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114650 dup_rows=131110 dup_rows_remote=114692 dedup_ratio=0.500145 router_tokens_per_expert=[0, 8182, 0, 8, 0, 0, 0, 0, 7456, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8178, 0, 0, 8192, 0, 700, 0, 7, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8175, 8185, 0, 48, 8190, 0, 5, 8189, 8192, 3, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16384, 16384, 16356, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14160, 14950, 16356, 12530, 16348, 23926, 16422, 16418] combine_rows_per_src=[16374, 16384, 16384, 16356, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=915464 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940439560 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879963656 weight_movement_bytes=352321536 total_nvlink_bytes=2232285192 weight_to_token_ratio=0.187409 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114732 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8186, 0, 6, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8138, 0, 0, 8191, 0, 14, 0, 10, 8192, 0, 0, 0, 8192, 8180, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 8179, 8189, 0, 68, 8192, 0, 15, 8188, 8192, 7, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16374, 16344, 16274, 16432, 16360, 16502, 16420, 16370] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=938885120 dispatch_meta_bytes=915368 combine_payload_bytes=938885120 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939800488 ingress_bytes=1115045888 link_bytes=1115045888 token_bytes=1878685608 weight_movement_bytes=176160768 total_nvlink_bytes=2054846376 weight_to_token_ratio=0.093768 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243791530e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114702 dedup_ratio=0.500053 router_tokens_per_expert=[0, 8183, 0, 34, 0, 0, 0, 0, 7474, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8186, 0, 0, 8189, 0, 662, 0, 5, 8192, 0, 0, 0, 8192, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8185, 8191, 0, 31, 8192, 0, 7, 8192, 8192, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16434, 14966, 16366, 17718, 16384, 16444, 16398, 16376] combine_rows_per_src=[16370, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114676 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 1, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8173, 0, 0, 8186, 0, 123, 0, 5, 8192, 0, 0, 0, 8192, 8185, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8186, 8190, 0, 56, 8192, 0, 7, 8188, 8192, 3, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 16086, 16334, 16640, 16370, 16484, 16396, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=917368 combine_payload_bytes=939212800 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940130168 ingress_bytes=1115373568 link_bytes=1115373568 token_bytes=1879342968 weight_movement_bytes=176160768 total_nvlink_bytes=2055503736 weight_to_token_ratio=0.093735 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245626401e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917664 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940408992 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900320 weight_movement_bytes=352321536 total_nvlink_bytes=2232221856 weight_to_token_ratio=0.187415 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917504 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=88080384 egress_bytes=940326912 ingress_bytes=1203650560 link_bytes=1203650560 token_bytes=1879736320 weight_movement_bytes=264241152 total_nvlink_bytes=2143977472 weight_to_token_ratio=0.140574 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.739940708e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=107704 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 3532, 8192, 8192, 0, 0, 0, 0, 8188, 0, 0, 14, 0, 8192, 0, 0, 0, 0, 3501, 8186, 8189, 0, 0, 1, 0, 6, 0, 8191, 0, 1326, 0, 0, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 17, 8192, 0, 8192, 8192, 0, 0, 8140, 0, 0, 0, 8190, 0, 8046, 10, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[7064, 16404, 23368, 16396, 19030, 16420, 16280, 16110] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114672 dup_rows=131090 dup_rows_remote=114652 dedup_ratio=0.500069 router_tokens_per_expert=[8181, 0, 0, 0, 5886, 8186, 8125, 0, 3, 3, 0, 8183, 7, 0, 8, 4, 8187, 1, 121, 0, 0, 1668, 8184, 8188, 0, 4, 1, 19, 8, 4, 8186, 4, 3491, 2, 4, 9, 11, 6547, 3, 5, 5, 2, 3, 4, 3, 0, 9, 8188, 2, 8184, 8183, 0, 4, 8087, 5, 0, 4, 8187, 0, 6959, 6, 1, 3, 0] payload_rows_per_dst=[16376, 16384, 16376, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[11646, 16404, 19948, 16438, 20122, 16418, 16178, 13936] combine_rows_per_src=[16376, 16384, 16376, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917456 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940441552 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965648 weight_movement_bytes=440401920 total_nvlink_bytes=2320367568 weight_to_token_ratio=0.234261 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 13, 0, 0, 0, 0, 8192, 6, 8172, 8192, 8192, 0, 0, 0, 0, 1, 0, 8192, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16422, 16346, 16384, 16384, 16384, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 2, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130504 payload_rows_remote=114120 dup_rows=131640 dup_rows_remote=113890 dedup_ratio=0.502167 router_tokens_per_expert=[7, 7670, 4, 1200, 0, 0, 9, 0, 6642, 0, 8, 0, 0, 395, 5, 11, 25, 0, 6, 8180, 6, 7, 8122, 13, 744, 9, 9, 8158, 0, 0, 6, 8185, 8174, 0, 0, 8160, 0, 3, 0, 64, 29, 8185, 8165, 7872, 7, 32, 8143, 0, 14, 8178, 8170, 2, 1, 50, 7, 0, 9, 8182, 8182, 0, 0, 11, 0, 11] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 15822, 16384, 16384] dup_rows_per_dst=[17750, 21068, 16708, 17702, 16426, 9360, 16222, 16404] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 15822, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=889568 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=940413664 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879937760 weight_movement_bytes=704643072 total_nvlink_bytes=2584580832 weight_to_token_ratio=0.374823 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.023674 gemm_active_groups=15 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=917296 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940310320 ingress_bytes=1115553792 link_bytes=1115553792 token_bytes=1879703344 weight_movement_bytes=176160768 total_nvlink_bytes=2055864112 weight_to_token_ratio=0.093717 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246635580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=107916 dedup_ratio=0.500008 router_tokens_per_expert=[8192, 0, 0, 0, 4576, 8192, 8192, 0, 0, 0, 0, 8173, 0, 0, 27, 0, 8192, 0, 0, 0, 0, 1613, 8189, 8187, 0, 0, 0, 0, 26, 0, 8192, 0, 2491, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 31, 8192, 0, 8192, 8192, 0, 0, 7784, 0, 0, 0, 8190, 0, 8053, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[9152, 16400, 9920, 16038, 21366, 16452, 18588, 23158] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114628 dedup_ratio=0.500015 router_tokens_per_expert=[8192, 0, 0, 0, 4861, 8192, 8192, 0, 0, 0, 0, 8176, 0, 0, 48, 0, 8192, 0, 0, 0, 0, 3024, 8186, 8180, 0, 0, 2, 0, 33, 0, 8192, 0, 637, 0, 0, 0, 0, 8192, 0, 0, 3, 8, 0, 0, 0, 0, 33, 8192, 0, 8192, 8192, 0, 0, 7931, 0, 0, 0, 8186, 0, 8032, 4, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[31076, 16448, 8872, 16454, 9828, 16472, 15862, 16064] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441600 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965696 weight_movement_bytes=528482304 total_nvlink_bytes=2408448000 weight_to_token_ratio=0.281113 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=934871040 dispatch_meta_bytes=912040 combine_payload_bytes=934871040 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=935783080 ingress_bytes=1111031808 link_bytes=1111031808 token_bytes=1870654120 weight_movement_bytes=176160768 total_nvlink_bytes=2046814888 weight_to_token_ratio=0.094171 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.221314357e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=110838 dedup_ratio=0.500008 router_tokens_per_expert=[8192, 0, 0, 0, 4501, 8192, 8192, 0, 0, 0, 0, 8183, 0, 0, 9, 0, 8192, 0, 0, 0, 0, 2156, 8186, 8186, 0, 0, 0, 0, 13, 0, 8192, 0, 1936, 0, 0, 0, 0, 8182, 0, 0, 7, 3, 0, 0, 0, 0, 5, 8192, 0, 8192, 8192, 0, 0, 8083, 0, 0, 0, 8191, 0, 7887, 8, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[9002, 16384, 20672, 16410, 20236, 16414, 16166, 15790] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=890416 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940414512 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879938608 weight_movement_bytes=176160768 total_nvlink_bytes=2056099376 weight_to_token_ratio=0.093706 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114880 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 5577, 8192, 8192, 0, 0, 0, 0, 8185, 0, 0, 16, 0, 8192, 0, 0, 1, 0, 2117, 8189, 8181, 0, 0, 1, 0, 15, 0, 8191, 0, 658, 0, 0, 0, 0, 8192, 0, 0, 3, 2, 0, 0, 0, 0, 7, 8192, 0, 8192, 8192, 0, 0, 8096, 0, 0, 0, 8191, 0, 8101, 4, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[11154, 16402, 20592, 16414, 17700, 16408, 16192, 16210] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917248 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940408576 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879899904 weight_movement_bytes=176160768 total_nvlink_bytes=2056060672 weight_to_token_ratio=0.093708 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114678 dedup_ratio=0.500008 router_tokens_per_expert=[7808, 8190, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8191, 0, 7, 0, 0, 8192, 0, 0, 0, 0, 1, 792, 8191, 0, 8192, 6070, 0, 0, 8192, 0, 8188, 0, 2, 0, 0, 1709, 2, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15616, 16382, 16396, 16386, 13724, 19802, 16384, 16384] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=902096 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940409808 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879917520 weight_movement_bytes=176160768 total_nvlink_bytes=2056078288 weight_to_token_ratio=0.093707 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114654 dup_rows=131106 dup_rows_remote=114712 dedup_ratio=0.500130 router_tokens_per_expert=[7436, 8166, 6, 3, 0, 3, 8184, 1, 5, 5, 5, 4, 4, 0, 8183, 3, 7983, 0, 4, 0, 0, 8188, 221, 7, 0, 0, 8188, 7, 4, 0, 1, 1, 638, 8184, 0, 8188, 7935, 5, 28, 8183, 101, 8186, 0, 2, 0, 7, 376, 2, 4, 8188, 0, 0, 4, 0, 8181, 0, 4, 4, 5, 0, 0, 8043, 8187, 5] payload_rows_per_dst=[16352, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14878, 16404, 16422, 16394, 17194, 17332, 16370, 16112] combine_rows_per_src=[16352, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918272 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940442368 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879966464 weight_movement_bytes=352321536 total_nvlink_bytes=2232288000 weight_to_token_ratio=0.187408 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114638 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 5, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8188, 0, 0, 8192, 0, 22, 0, 0, 8192, 0, 0, 0, 8192, 8188, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 8191, 0, 26, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16394, 16296, 16376, 16428, 16376, 16434, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130724 payload_rows_remote=114354 dup_rows=131420 dup_rows_remote=88022 dedup_ratio=0.501328 router_tokens_per_expert=[8174, 0, 0, 0, 5303, 8181, 8117, 0, 6, 7, 0, 8177, 15, 0, 23, 6, 8183, 0, 289, 0, 2, 2820, 8183, 8185, 0, 7, 0, 37, 26, 7, 8184, 7, 3461, 0, 7, 15, 22, 6190, 5, 8, 0, 2, 2, 7, 7, 0, 6, 8185, 6, 8180, 8176, 0, 7, 7883, 10, 0, 7, 8185, 0, 6743, 14, 0, 5, 0] payload_rows_per_dst=[16370, 16384, 16370, 16382, 16066, 16384, 16384, 16384] dup_rows_per_dst=[43398, 16232, 6536, 16514, 3350, 16086, 15780, 13524] combine_rows_per_src=[16370, 16384, 16370, 16382, 16066, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130992 payload_rows_remote=114608 dup_rows=131152 dup_rows_remote=114094 dedup_ratio=0.500305 router_tokens_per_expert=[7786, 8185, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8191, 0, 31, 0, 0, 8192, 0, 0, 0, 0, 2, 376, 8192, 0, 8192, 7785, 0, 0, 8192, 0, 8180, 0, 4, 0, 0, 414, 7, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 8192, 0] payload_rows_per_dst=[16304, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[9132, 16342, 16416, 16366, 22216, 17210, 16412, 17058] combine_rows_per_src=[16304, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[2, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917456 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940425168 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879932880 weight_movement_bytes=352321536 total_nvlink_bytes=2232254416 weight_to_token_ratio=0.187412 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=917464 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940163032 ingress_bytes=1115406336 link_bytes=1115406336 token_bytes=1879408600 weight_movement_bytes=176160768 total_nvlink_bytes=2055569368 weight_to_token_ratio=0.093732 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245809888e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=114716 dedup_ratio=0.500107 router_tokens_per_expert=[7862, 8178, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8188, 0, 22, 0, 0, 8192, 0, 0, 0, 0, 4, 186, 8192, 0, 8192, 7774, 0, 0, 8192, 0, 8177, 0, 7, 0, 0, 551, 11, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16356, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15724, 16384, 16420, 16392, 15920, 17492, 16384, 16384] combine_rows_per_src=[16356, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917304 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940441400 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879965496 weight_movement_bytes=704643072 total_nvlink_bytes=2584608568 weight_to_token_ratio=0.374817 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=936787968 dispatch_meta_bytes=809504 combine_payload_bytes=936787968 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=937597472 ingress_bytes=1465270272 link_bytes=1465270272 token_bytes=1874385440 weight_movement_bytes=528482304 total_nvlink_bytes=2402867744 weight_to_token_ratio=0.281950 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.024621 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.204901889e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=116598 dedup_ratio=0.500069 router_tokens_per_expert=[7917, 8183, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 11, 0, 0, 8192, 0, 0, 0, 0, 0, 638, 8192, 0, 8192, 6608, 0, 0, 8192, 0, 8186, 0, 1, 0, 0, 1219, 5, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16366, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15834, 16384, 16406, 16384, 14492, 18822, 16384, 16384] combine_rows_per_src=[16366, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938868736 dispatch_meta_bytes=914808 combine_payload_bytes=938868736 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939783544 ingress_bytes=1115029504 link_bytes=1115029504 token_bytes=1878652280 weight_movement_bytes=176160768 total_nvlink_bytes=2054813048 weight_to_token_ratio=0.093770 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.243699786e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114702 dedup_ratio=0.500053 router_tokens_per_expert=[7876, 8184, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8190, 0, 11, 0, 0, 8192, 0, 0, 0, 0, 2, 487, 8191, 0, 8192, 6774, 0, 0, 8192, 0, 8184, 0, 3, 0, 0, 1244, 5, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16370, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15752, 16382, 16402, 16388, 14522, 18872, 16384, 16384] combine_rows_per_src=[16370, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=917504 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940212224 ingress_bytes=1115455488 link_bytes=1115455488 token_bytes=1879506944 weight_movement_bytes=176160768 total_nvlink_bytes=2055667712 weight_to_token_ratio=0.093727 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246085119e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=925072 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940301712 ingress_bytes=1467858944 link_bytes=1467858944 token_bytes=1879678352 weight_movement_bytes=528482304 total_nvlink_bytes=2408160656 weight_to_token_ratio=0.281156 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219397372e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130018 payload_rows_remote=113640 dup_rows=132126 dup_rows_remote=115754 dedup_ratio=0.504021 router_tokens_per_expert=[14, 8187, 4, 4293, 3, 172, 3, 0, 8187, 4, 8182, 369, 7, 0, 0, 0, 8187, 4, 136, 8109, 3, 8184, 0, 12, 4, 1, 10, 1, 0, 8174, 0, 8185, 0, 4, 3, 8113, 1, 3698, 0, 0, 6, 12, 5, 0, 0, 331, 5, 3, 14, 5, 8052, 0, 0, 6, 8187, 7655, 0, 8163, 0, 8183, 8177, 0, 9, 5] payload_rows_per_dst=[16384, 16384, 16384, 16378, 16384, 16384, 15346, 16374] dup_rows_per_dst=[19832, 17114, 16510, 16372, 23628, 17082, 5254, 16334] combine_rows_per_src=[16384, 16384, 16384, 16378, 16384, 16384, 15346, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114690 dedup_ratio=0.500008 router_tokens_per_expert=[0, 8192, 0, 3911, 0, 0, 2, 0, 8191, 0, 8138, 1300, 0, 0, 0, 0, 8191, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 0, 8074, 1, 3142, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8191, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[24210, 18874, 16384, 16386, 22434, 16402, 0, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917504 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940326912 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879736320 weight_movement_bytes=176160768 total_nvlink_bytes=2055897088 weight_to_token_ratio=0.093716 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114670 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 4301, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 8192, 0, 0, 0, 0, 2402, 8192, 8191, 0, 0, 0, 0, 14, 0, 8192, 0, 1687, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 5, 8192, 0, 8192, 8192, 0, 0, 8013, 0, 0, 0, 8192, 0, 8144, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8602, 16398, 21186, 16412, 19758, 16402, 16026, 16288] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114640 dup_rows=131120 dup_rows_remote=102572 dedup_ratio=0.500183 router_tokens_per_expert=[7144, 8162, 10, 2, 0, 5, 8180, 0, 9, 8, 8, 6, 6, 0, 8163, 14, 8035, 0, 7, 0, 0, 8186, 192, 21, 0, 0, 8185, 11, 1, 0, 0, 0, 1229, 8184, 0, 8184, 7757, 8, 46, 8178, 146, 8173, 0, 0, 2, 12, 232, 6, 7, 8185, 0, 4, 6, 0, 8175, 0, 7, 7, 7, 0, 0, 7971, 8183, 8] payload_rows_per_dst=[16384, 16384, 16336, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[28548, 16412, 16268, 16378, 18052, 3110, 16370, 15982] combine_rows_per_src=[16384, 16384, 16336, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130538 payload_rows_remote=114154 dup_rows=131606 dup_rows_remote=112894 dedup_ratio=0.502037 router_tokens_per_expert=[0, 8192, 0, 5125, 0, 0, 19, 0, 8192, 0, 8164, 1216, 0, 0, 0, 0, 8189, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 0, 7719, 2, 2338, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8173, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 15852, 16384, 16384, 16384] dup_rows_per_dst=[10288, 7480, 14056, 16386, 4266, 16406, 44012, 18712] combine_rows_per_src=[16384, 16382, 16384, 16384, 15852, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=930938880 dispatch_meta_bytes=917576 combine_payload_bytes=930938880 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=931856456 ingress_bytes=1107099648 link_bytes=1107099648 token_bytes=1862795336 weight_movement_bytes=176160768 total_nvlink_bytes=2038956104 weight_to_token_ratio=0.094568 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.199295903e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917504 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940425216 ingress_bytes=1467990016 link_bytes=1467990016 token_bytes=1879932928 weight_movement_bytes=528482304 total_nvlink_bytes=2408415232 weight_to_token_ratio=0.281118 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220131320e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917432 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441528 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965624 weight_movement_bytes=176160768 total_nvlink_bytes=2056126392 weight_to_token_ratio=0.093704 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114070 dedup_ratio=0.500023 router_tokens_per_expert=[0, 8192, 0, 3630, 0, 0, 15, 0, 8192, 0, 8155, 349, 0, 0, 0, 0, 8191, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8007, 1, 4406, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8189, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16378, 16384] dup_rows_per_dst=[23674, 17008, 16384, 16384, 24828, 16416, 0, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114680 dup_rows=131086 dup_rows_remote=116930 dedup_ratio=0.500053 router_tokens_per_expert=[0, 8192, 0, 5990, 0, 0, 13, 0, 8192, 0, 8100, 320, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8107, 0, 2042, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8184, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16376, 16384] dup_rows_per_dst=[14080, 16840, 16384, 16384, 14156, 16408, 20450, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=868848 combine_payload_bytes=939130880 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939999728 ingress_bytes=1643773952 link_bytes=1643773952 token_bytes=1879130608 weight_movement_bytes=704643072 total_nvlink_bytes=2583773680 weight_to_token_ratio=0.374984 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204447986e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=935149568 dispatch_meta_bytes=908192 combine_payload_bytes=935149568 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=936057760 ingress_bytes=1463631872 link_bytes=1463631872 token_bytes=1871207328 weight_movement_bytes=528482304 total_nvlink_bytes=2399689632 weight_to_token_ratio=0.282429 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.195727533e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29927941552 prefetch_bytes=2906652672 reduce_bytes=3963617280 weight_movement_bytes=6870269952 egress_bytes=14971315632 ingress_bytes=21826895872 link_bytes=21826895872 total_nvlink_bytes=36798211504 weight_to_token_ratio=0.229560 prefetch_share_of_ingress=0.133168 dedup_ratio=0.502590 prefetch_slots=66 unused_prefetch_slots=446 slot_utilization=0.128906 reduce_slots=45 remote_demand=33 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021070 gemm_inner_pad_upper_bound=33274 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.638844118e-05 prefetch_bytes_per_gemm_flop=1.017252604e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29897875040 prefetch_bytes=2906652672 reduce_bytes=3963617280 weight_movement_bytes=6870269952 egress_bytes=14956224096 ingress_bytes=21811920896 link_bytes=21811920896 total_nvlink_bytes=36768144992 weight_to_token_ratio=0.229791 prefetch_share_of_ingress=0.133260 dedup_ratio=0.502746 prefetch_slots=66 unused_prefetch_slots=446 slot_utilization=0.128906 reduce_slots=45 remote_demand=33 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021188 gemm_inner_pad_upper_bound=33274 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.633603267e-05 prefetch_bytes_per_gemm_flop=1.017252604e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=98320 dedup_ratio=0.500046 router_tokens_per_expert=[0, 8192, 0, 2473, 0, 0, 6, 0, 8191, 0, 8161, 531, 0, 0, 0, 0, 8191, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 0, 8143, 1, 5257, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8190, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16372, 16384, 16384, 16384] dup_rows_per_dst=[4958, 17382, 16384, 16386, 10430, 16396, 32764, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16372, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=915008 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940389952 ingress_bytes=1291796480 link_bytes=1291796480 token_bytes=1879864896 weight_movement_bytes=352321536 total_nvlink_bytes=2232186432 weight_to_token_ratio=0.187419 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233521066e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29706265216 prefetch_bytes=1937768448 reduce_bytes=3258974208 weight_movement_bytes=5196742656 egress_bytes=14860362368 ingress_bytes=20042645504 link_bytes=20042645504 total_nvlink_bytes=34903007872 weight_to_token_ratio=0.174938 prefetch_share_of_ingress=0.096682 dedup_ratio=0.505627 prefetch_slots=44 unused_prefetch_slots=468 slot_utilization=0.085938 reduce_slots=37 remote_demand=22 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021248 gemm_inner_pad_upper_bound=32639 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.014403038e-05 prefetch_bytes_per_gemm_flop=6.781684028e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=926440 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940385000 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879843560 weight_movement_bytes=176160768 total_nvlink_bytes=2056004328 weight_to_token_ratio=0.093710 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=851984 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940277776 ingress_bytes=1467908096 link_bytes=1467908096 token_bytes=1879703568 weight_movement_bytes=528482304 total_nvlink_bytes=2408185872 weight_to_token_ratio=0.281152 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219672603e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=112640 dedup_ratio=0.500000 router_tokens_per_expert=[7881, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 8192, 0, 0, 0, 0, 0, 283, 8192, 0, 8192, 7193, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 1024, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15762, 16384, 16390, 16384, 14952, 18432, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29881407152 prefetch_bytes=2994733056 reduce_bytes=2378170368 weight_movement_bytes=5372903424 egress_bytes=14947997360 ingress_bytes=20306313216 link_bytes=20306313216 total_nvlink_bytes=35254310576 weight_to_token_ratio=0.179808 prefetch_share_of_ingress=0.147478 dedup_ratio=0.503025 prefetch_slots=68 unused_prefetch_slots=444 slot_utilization=0.132812 reduce_slots=27 remote_demand=34 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021307 gemm_inner_pad_upper_bound=32766 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.106679859e-05 prefetch_bytes_per_gemm_flop=1.048078441e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29961128952 prefetch_bytes=3258974208 reduce_bytes=2466250752 weight_movement_bytes=5725224960 egress_bytes=14987922424 ingress_bytes=20698431488 link_bytes=20698431488 total_nvlink_bytes=35686353912 weight_to_token_ratio=0.191088 prefetch_share_of_ingress=0.157450 dedup_ratio=0.502022 prefetch_slots=74 unused_prefetch_slots=438 slot_utilization=0.144531 reduce_slots=28 remote_demand=37 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=34417 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.243911024e-05 prefetch_bytes_per_gemm_flop=1.140555950e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129422 payload_rows_remote=113038 dup_rows=132722 dup_rows_remote=107918 dedup_ratio=0.506294 router_tokens_per_expert=[28, 8185, 6, 3930, 4, 244, 13, 0, 8185, 6, 8181, 455, 12, 0, 0, 0, 8183, 4, 247, 8052, 6, 8180, 1, 22, 2, 4, 14, 0, 0, 8159, 0, 8181, 0, 7, 5, 8102, 0, 3924, 0, 0, 9, 1, 8, 0, 0, 559, 8, 6, 28, 9, 7995, 0, 0, 11, 8184, 7410, 0, 8136, 1, 8179, 8166, 0, 13, 7] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16384, 16384, 14888, 16238] dup_rows_per_dst=[24804, 19380, 18586, 16404, 23682, 15576, 28, 14262] combine_rows_per_src=[16384, 16384, 16384, 16376, 16384, 16384, 14888, 16238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29885098720 prefetch_bytes=2642411520 reduce_bytes=2554331136 weight_movement_bytes=5196742656 egress_bytes=14949821152 ingress_bytes=20132020224 link_bytes=20132020224 total_nvlink_bytes=35081841376 weight_to_token_ratio=0.173891 prefetch_share_of_ingress=0.131254 dedup_ratio=0.502976 prefetch_slots=60 unused_prefetch_slots=452 slot_utilization=0.117188 reduce_slots=29 remote_demand=30 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021484 gemm_inner_pad_upper_bound=32512 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.045681858e-05 prefetch_bytes_per_gemm_flop=9.247750947e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=909312 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940433408 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879957504 weight_movement_bytes=176160768 total_nvlink_bytes=2056118272 weight_to_token_ratio=0.093705 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=926007296 dispatch_meta_bytes=883824 combine_payload_bytes=926007296 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=926891120 ingress_bytes=1102168064 link_bytes=1102168064 token_bytes=1852898416 weight_movement_bytes=176160768 total_nvlink_bytes=2029059184 weight_to_token_ratio=0.095073 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.171681091e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29812294936 prefetch_bytes=2906652672 reduce_bytes=1849688064 weight_movement_bytes=4756340736 egress_bytes=14913340696 ingress_bytes=19655294976 link_bytes=19655294976 total_nvlink_bytes=34568635672 weight_to_token_ratio=0.159543 prefetch_share_of_ingress=0.147881 dedup_ratio=0.503983 prefetch_slots=66 unused_prefetch_slots=446 slot_utilization=0.128906 reduce_slots=21 remote_demand=33 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=33528 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.878840459e-05 prefetch_bytes_per_gemm_flop=1.017252604e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114692 dedup_ratio=0.500023 router_tokens_per_expert=[0, 8192, 0, 4605, 0, 0, 8, 0, 8192, 0, 8161, 337, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8162, 0, 3302, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16384] dup_rows_per_dst=[9226, 16996, 16384, 16384, 8204, 16386, 31114, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=917496 combine_payload_bytes=939474944 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940392440 ingress_bytes=1291796480 link_bytes=1291796480 token_bytes=1879867384 weight_movement_bytes=352321536 total_nvlink_bytes=2232188920 weight_to_token_ratio=0.187418 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233521066e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29949981040 prefetch_bytes=3170893824 reduce_bytes=2290089984 weight_movement_bytes=5460983808 egress_bytes=14982312304 ingress_bytes=20428652544 link_bytes=20428652544 total_nvlink_bytes=35410964848 weight_to_token_ratio=0.182337 prefetch_share_of_ingress=0.155218 dedup_ratio=0.502186 prefetch_slots=72 unused_prefetch_slots=440 slot_utilization=0.140625 reduce_slots=26 remote_demand=36 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=34417 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.149495432e-05 prefetch_bytes_per_gemm_flop=1.109730114e-05 anomalies=none - [2026-09-16 21:39:34] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 609.2 | throughput per GPU (TFLOP/s/GPU): 547.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.112984E+00 | loss scale: 1.0 | grad norm: 5.426 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121750 payload_rows_remote=106344 dup_rows=140394 dup_rows_remote=124094 dedup_ratio=0.535561 router_tokens_per_expert=[4887, 670, 1435, 1171, 3787, 0, 2527, 827, 1485, 5217, 2618, 1576, 2546, 2940, 0, 444, 1110, 2057, 3313, 2261, 3, 871, 2558, 2028, 192, 91, 5576, 2200, 4525, 1513, 4955, 2377, 77, 3381, 4145, 553, 1590, 910, 4477, 2849, 2565, 137, 3957, 2647, 1, 3580, 1828, 1971, 5976, 79, 1708, 14, 266, 2918, 12, 1981, 0, 91, 462, 1960, 4891, 1813, 2427, 4046] payload_rows_per_dst=[14796, 15588, 14468, 15406, 15574, 14730, 16060, 15128] dup_rows_per_dst=[15812, 18064, 13934, 16300, 20390, 18642, 21000, 16252] combine_rows_per_src=[14796, 15588, 14468, 15406, 15574, 14730, 16060, 15128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121770 payload_rows_remote=105938 dup_rows=140374 dup_rows_remote=121018 dedup_ratio=0.535484 router_tokens_per_expert=[5058, 600, 1128, 728, 2998, 0, 2415, 1027, 2628, 6113, 3591, 1428, 2498, 3589, 0, 542, 970, 2440, 2182, 2019, 2, 1227, 3683, 2580, 155, 211, 5040, 1897, 5100, 2310, 4608, 1961, 51, 3703, 4696, 652, 1231, 649, 3816, 2256, 2008, 248, 3774, 3297, 0, 3409, 1548, 2164, 5218, 133, 2290, 28, 317, 3255, 20, 1293, 0, 110, 556, 1231, 4617, 1224, 2002, 4548] payload_rows_per_dst=[14236, 15764, 15602, 15418, 15450, 14772, 15832, 14696] dup_rows_per_dst=[13672, 19360, 20258, 17066, 18658, 18124, 19356, 13880] combine_rows_per_src=[14236, 15764, 15602, 15418, 15450, 14772, 15832, 14696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121344 payload_rows_remote=106112 dup_rows=140800 dup_rows_remote=122390 dedup_ratio=0.537109 router_tokens_per_expert=[5330, 663, 1869, 1022, 3317, 0, 3028, 741, 2276, 5431, 3162, 1163, 2532, 2571, 0, 403, 1167, 1808, 2914, 2369, 18, 1313, 3285, 2050, 358, 114, 5863, 1832, 4516, 1408, 4847, 2215, 123, 3103, 4477, 276, 967, 769, 3782, 2630, 2325, 308, 3502, 3220, 0, 3725, 1521, 1679, 5318, 71, 2306, 32, 640, 2671, 10, 1401, 10, 163, 929, 1975, 4928, 2040, 1830, 4756] payload_rows_per_dst=[15488, 15204, 15232, 15090, 15266, 14598, 15074, 15392] dup_rows_per_dst=[21014, 15310, 18410, 16676, 16988, 17962, 16570, 17870] combine_rows_per_src=[15488, 15204, 15232, 15090, 15266, 14598, 15074, 15392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121504 payload_rows_remote=106860 dup_rows=140640 dup_rows_remote=123028 dedup_ratio=0.536499 router_tokens_per_expert=[5502, 512, 1779, 992, 3514, 0, 2845, 654, 1915, 5685, 3439, 1235, 2629, 2501, 1, 475, 870, 2134, 2013, 1940, 5, 1051, 3441, 1979, 208, 73, 5581, 2637, 5516, 1663, 5195, 1992, 38, 3822, 4648, 680, 1087, 794, 3656, 2413, 1893, 204, 3679, 2783, 0, 3590, 2087, 1892, 5242, 118, 2002, 6, 443, 3043, 6, 1223, 1, 134, 709, 1695, 5048, 1394, 1910, 4856] payload_rows_per_dst=[14888, 14964, 14758, 15718, 15304, 14644, 15940, 15288] dup_rows_per_dst=[16708, 17868, 15036, 18850, 18972, 17612, 19388, 16206] combine_rows_per_src=[14888, 14964, 14758, 15718, 15304, 14644, 15940, 15288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121354 payload_rows_remote=105734 dup_rows=140790 dup_rows_remote=120100 dedup_ratio=0.537071 router_tokens_per_expert=[5064, 529, 1063, 1001, 3121, 0, 2832, 653, 1713, 5586, 2834, 2353, 3083, 2716, 11, 603, 776, 1794, 3394, 1996, 2, 868, 4353, 2171, 211, 130, 5398, 2014, 5089, 1886, 4445, 2207, 76, 2900, 4913, 438, 1109, 926, 3836, 1969, 2479, 145, 3868, 2276, 4, 3582, 1642, 1738, 5977, 149, 1962, 13, 468, 3192, 36, 1309, 3, 137, 595, 1864, 5433, 1473, 1860, 4804] payload_rows_per_dst=[14610, 15256, 15716, 14970, 14284, 14768, 16130, 15620] dup_rows_per_dst=[13916, 16442, 21092, 14854, 13308, 19610, 20878, 20690] combine_rows_per_src=[14610, 15256, 15716, 14970, 14284, 14768, 16130, 15620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120488 payload_rows_remote=105950 dup_rows=141656 dup_rows_remote=127482 dedup_ratio=0.540375 router_tokens_per_expert=[5005, 540, 1523, 754, 4171, 2, 3557, 546, 1577, 5203, 3079, 1600, 2693, 2247, 0, 301, 680, 2190, 2845, 1904, 15, 1063, 3650, 2155, 269, 63, 5808, 2252, 5240, 1479, 4700, 2414, 86, 3384, 3816, 399, 994, 1019, 3993, 3049, 2108, 244, 4002, 2683, 3, 3835, 1635, 2113, 5590, 149, 2033, 8, 269, 2845, 0, 1333, 4, 157, 641, 1737, 5282, 1420, 2082, 4634] payload_rows_per_dst=[15622, 14538, 14860, 15698, 15336, 15036, 14156, 15242] dup_rows_per_dst=[22474, 14174, 19260, 22424, 18144, 18210, 10298, 16672] combine_rows_per_src=[15622, 14538, 14860, 15698, 15336, 15036, 14156, 15242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120312 payload_rows_remote=104400 dup_rows=141832 dup_rows_remote=118046 dedup_ratio=0.541046 router_tokens_per_expert=[4729, 690, 1999, 1107, 3288, 5, 2311, 554, 1720, 4794, 2865, 1381, 3453, 1705, 0, 469, 797, 2507, 3228, 1924, 5, 998, 3722, 3024, 201, 47, 5300, 2251, 4883, 1465, 5241, 2623, 170, 4471, 4425, 512, 922, 608, 3581, 2059, 2116, 158, 3854, 2956, 13, 3201, 2258, 1426, 5052, 70, 1933, 39, 343, 2696, 20, 1766, 7, 152, 795, 2604, 5043, 1741, 1971, 4824] payload_rows_per_dst=[15912, 15030, 15538, 15382, 14744, 14236, 14110, 15360] dup_rows_per_dst=[23786, 17408, 22054, 17570, 15562, 16810, 9728, 18914] combine_rows_per_src=[15912, 15030, 15538, 15382, 14744, 14236, 14110, 15360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=871170048 dispatch_meta_bytes=921752 combine_payload_bytes=871170048 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=872091800 ingress_bytes=1575813120 link_bytes=1575813120 token_bytes=1743261848 weight_movement_bytes=704643072 total_nvlink_bytes=2447904920 weight_to_token_ratio=0.404210 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=18944 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.823895695e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=121980 payload_rows_remote=106524 dup_rows=140164 dup_rows_remote=121456 dedup_ratio=0.534683 router_tokens_per_expert=[4177, 594, 1511, 922, 3406, 1, 3765, 229, 1082, 6537, 3533, 1165, 2522, 3497, 1, 250, 667, 2511, 2415, 1595, 4, 1102, 2612, 2057, 184, 110, 5928, 1817, 4639, 1518, 4650, 2443, 83, 3163, 4337, 514, 975, 771, 3883, 3356, 1764, 130, 4101, 2531, 0, 3962, 2104, 1603, 5408, 126, 1438, 2, 465, 3710, 1, 1734, 0, 102, 345, 3031, 4819, 2502, 2011, 4657] payload_rows_per_dst=[14710, 16024, 14306, 15292, 15456, 14574, 16102, 15516] dup_rows_per_dst=[14500, 21150, 11620, 15430, 18708, 17816, 21522, 19418] combine_rows_per_src=[14710, 16024, 14306, 15292, 15456, 14574, 16102, 15516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=867844096 dispatch_meta_bytes=907824 combine_payload_bytes=867844096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=868751920 ingress_bytes=1044004864 link_bytes=1044004864 token_bytes=1736596016 weight_movement_bytes=176160768 total_nvlink_bytes=1912756784 weight_to_token_ratio=0.101440 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=22272 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.845991449e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=869269504 dispatch_meta_bytes=914008 combine_payload_bytes=869269504 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=870183512 ingress_bytes=1045430272 link_bytes=1045430272 token_bytes=1739453016 weight_movement_bytes=176160768 total_nvlink_bytes=1915613784 weight_to_token_ratio=0.101274 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=13952 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.853973139e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=875397120 dispatch_meta_bytes=919552 combine_payload_bytes=875397120 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=876316672 ingress_bytes=1051557888 link_bytes=1051557888 token_bytes=1751713792 weight_movement_bytes=176160768 total_nvlink_bytes=1927874560 weight_to_token_ratio=0.100565 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=15232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.888285231e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=866172928 dispatch_meta_bytes=903336 combine_payload_bytes=866172928 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=867076264 ingress_bytes=1042333696 link_bytes=1042333696 token_bytes=1733249192 weight_movement_bytes=176160768 total_nvlink_bytes=1909409960 weight_to_token_ratio=0.101636 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=20480 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.836633606e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=867942400 dispatch_meta_bytes=933728 combine_payload_bytes=867942400 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=868876128 ingress_bytes=1220263936 link_bytes=1220263936 token_bytes=1736818528 weight_movement_bytes=352321536 total_nvlink_bytes=2089140064 weight_to_token_ratio=0.202855 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=15104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.832968678e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=855244800 dispatch_meta_bytes=889784 combine_payload_bytes=855244800 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=856134584 ingress_bytes=1031405568 link_bytes=1031405568 token_bytes=1711379384 weight_movement_bytes=176160768 total_nvlink_bytes=1887540152 weight_to_token_ratio=0.102935 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=21248 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.775440651e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=872644608 dispatch_meta_bytes=911920 combine_payload_bytes=872644608 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=873556528 ingress_bytes=1401126912 link_bytes=1401126912 token_bytes=1746201136 weight_movement_bytes=528482304 total_nvlink_bytes=2274683440 weight_to_token_ratio=0.302647 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=16896 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.845725848e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114674 dup_rows=131094 dup_rows_remote=112544 dedup_ratio=0.500084 router_tokens_per_expert=[0, 0, 0, 1, 8190, 7, 7987, 1, 8190, 148, 1, 432, 8157, 0, 0, 7669, 3152, 1, 0, 2, 0, 1, 8190, 1219, 1, 1530, 0, 3155, 0, 6354, 8184, 1628, 0, 0, 3, 1, 251, 0, 0, 2276, 8188, 6623, 1, 0, 1, 0, 0, 8189, 1, 1, 1, 7236, 8186, 1, 2, 1, 60, 8190, 5251, 1159, 1, 0, 1, 1248] payload_rows_per_dst=[16384, 16378, 16384, 16376, 16384, 16384, 16376, 16384] dup_rows_per_dst=[15988, 16494, 11654, 18550, 19970, 18520, 14482, 15436] combine_rows_per_src=[16384, 16378, 16384, 16376, 16384, 16384, 16376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128888 payload_rows_remote=112506 dup_rows=133256 dup_rows_remote=110954 dedup_ratio=0.508331 router_tokens_per_expert=[0, 0, 0, 0, 8192, 1, 7757, 0, 8190, 68, 0, 554, 8170, 0, 0, 7260, 4961, 0, 0, 0, 0, 0, 8192, 1285, 0, 2544, 0, 3250, 0, 6593, 8155, 1295, 0, 0, 1, 0, 457, 1, 1, 1186, 8178, 6577, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 5789, 8191, 0, 1, 0, 202, 8192, 4560, 1769, 0, 0, 0, 1382] payload_rows_per_dst=[16384, 14782, 16384, 15814, 16374, 16384, 16382, 16384] dup_rows_per_dst=[15516, 9140, 12492, 12598, 18598, 20366, 22302, 22244] combine_rows_per_src=[16384, 14782, 16384, 15814, 16374, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=109476 dedup_ratio=0.500084 router_tokens_per_expert=[0, 1, 0, 0, 8192, 2, 7339, 0, 8184, 135, 0, 190, 8170, 0, 0, 6260, 4663, 0, 0, 0, 0, 0, 8192, 991, 0, 3017, 0, 4145, 1, 5430, 8184, 1865, 0, 0, 0, 0, 1001, 0, 2, 2392, 8192, 5696, 0, 0, 0, 0, 0, 8159, 11, 0, 0, 7006, 8192, 0, 0, 0, 62, 8190, 5267, 413, 0, 0, 0, 1528] payload_rows_per_dst=[16384, 16380, 16384, 16382, 16384, 16370, 16384, 16382] dup_rows_per_dst=[14684, 19184, 21618, 22158, 13538, 11340, 14034, 14538] combine_rows_per_src=[16384, 16380, 16384, 16382, 16384, 16370, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129158 payload_rows_remote=112774 dup_rows=132986 dup_rows_remote=111922 dedup_ratio=0.507301 router_tokens_per_expert=[0, 0, 0, 0, 8190, 0, 7370, 0, 8187, 313, 0, 170, 8185, 0, 0, 6318, 4543, 0, 0, 0, 0, 0, 8158, 1987, 0, 3003, 0, 3937, 0, 5348, 8182, 1216, 0, 0, 0, 0, 948, 0, 0, 3181, 8183, 7665, 0, 0, 0, 0, 0, 8004, 1, 0, 0, 6218, 8192, 0, 13, 0, 31, 8192, 3466, 674, 0, 0, 0, 1197] payload_rows_per_dst=[16380, 15968, 16316, 14974, 16384, 16368, 16384, 16384] dup_rows_per_dst=[14740, 5820, 13060, 14836, 21820, 15504, 26142, 21064] combine_rows_per_src=[16380, 15968, 16316, 14974, 16384, 16368, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130958 payload_rows_remote=114574 dup_rows=131186 dup_rows_remote=108508 dedup_ratio=0.500435 router_tokens_per_expert=[3, 1, 0, 0, 8189, 0, 7814, 0, 8177, 210, 2, 375, 8165, 0, 2, 7445, 4010, 1, 0, 0, 0, 0, 8188, 1030, 1, 2172, 1, 3261, 7, 6431, 8178, 1331, 0, 1, 3, 1, 401, 1, 0, 1993, 8117, 6590, 0, 0, 0, 1, 0, 8175, 2, 0, 1, 6456, 8188, 1, 10, 0, 12, 8190, 5686, 870, 2, 2, 2, 1373] payload_rows_per_dst=[16384, 16312, 16382, 16348, 16382, 16384, 16382, 16384] dup_rows_per_dst=[15630, 14636, 10076, 19204, 19034, 22678, 14038, 15890] combine_rows_per_src=[16384, 16312, 16382, 16348, 16382, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130574 payload_rows_remote=114194 dup_rows=131570 dup_rows_remote=103534 dedup_ratio=0.501900 router_tokens_per_expert=[0, 0, 0, 1, 8188, 2, 7049, 0, 8179, 29, 0, 120, 8175, 0, 0, 6074, 3863, 1, 1, 2, 1, 1, 8190, 1277, 1, 4035, 1, 4163, 0, 6327, 8182, 1983, 6, 1, 3, 1, 716, 1, 5, 2768, 8176, 6361, 0, 0, 0, 1, 2, 8187, 1, 1, 0, 6451, 8189, 1, 5, 1, 130, 8190, 4187, 888, 1, 0, 1, 953] payload_rows_per_dst=[16382, 16380, 16384, 15898, 16382, 16382, 16382, 16384] dup_rows_per_dst=[14098, 28036, 24148, 13380, 13956, 12720, 12916, 12316] combine_rows_per_src=[16382, 16380, 16384, 15898, 16382, 16382, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=908872 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940318280 ingress_bytes=1555972096 link_bytes=1555972096 token_bytes=1879727688 weight_movement_bytes=616562688 total_nvlink_bytes=2496290376 weight_to_token_ratio=0.328006 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=24960 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.712794243e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=921649152 dispatch_meta_bytes=893840 combine_payload_bytes=921649152 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=922542992 ingress_bytes=1097809920 link_bytes=1097809920 token_bytes=1844192144 weight_movement_bytes=176160768 total_nvlink_bytes=2020352912 weight_to_token_ratio=0.095522 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.147277304e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130138 payload_rows_remote=113756 dup_rows=132006 dup_rows_remote=107308 dedup_ratio=0.503563 router_tokens_per_expert=[0, 0, 0, 0, 8191, 14, 7691, 0, 8184, 261, 0, 11, 8184, 1, 1, 5606, 3911, 1, 5, 3, 0, 0, 8163, 3339, 0, 1052, 0, 4389, 6, 6521, 8144, 2050, 1, 0, 9, 1, 807, 6, 3, 1288, 8152, 7200, 0, 0, 0, 1, 4, 8057, 10, 5, 0, 7161, 8186, 1, 19, 0, 216, 8176, 4168, 524, 1, 9, 0, 1339] payload_rows_per_dst=[16382, 16382, 16370, 15886, 16374, 15978, 16382, 16384] dup_rows_per_dst=[24698, 23174, 29616, 14376, 4276, 9002, 14382, 12482] combine_rows_per_src=[16382, 16382, 16370, 15886, 16374, 15978, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=896568 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940240440 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879584312 weight_movement_bytes=176160768 total_nvlink_bytes=2055745080 weight_to_token_ratio=0.093723 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=111194 dedup_ratio=0.500099 router_tokens_per_expert=[1, 1, 1, 0, 8190, 3, 7858, 1, 8191, 71, 0, 1004, 8187, 0, 0, 7284, 2183, 0, 6, 0, 0, 0, 8190, 2059, 0, 1719, 0, 2984, 0, 7521, 8175, 1002, 1, 0, 1, 0, 205, 0, 0, 1935, 8180, 6605, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 6395, 8192, 0, 1, 0, 44, 8192, 5067, 1652, 0, 0, 0, 1789] payload_rows_per_dst=[16382, 16362, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15728, 16716, 8494, 18980, 19904, 21380, 12792, 17104] combine_rows_per_src=[16382, 16362, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=923844608 dispatch_meta_bytes=898784 combine_payload_bytes=923844608 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=924743392 ingress_bytes=1100005376 link_bytes=1100005376 token_bytes=1848588000 weight_movement_bytes=176160768 total_nvlink_bytes=2024748768 weight_to_token_ratio=0.095295 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.022727 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.159570941e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=938590208 dispatch_meta_bytes=892328 combine_payload_bytes=938590208 prefetch_bytes=0 reduce_bytes=616562688 egress_bytes=939482536 ingress_bytes=1555152896 link_bytes=1555152896 token_bytes=1878072744 weight_movement_bytes=616562688 total_nvlink_bytes=2494635432 weight_to_token_ratio=0.328295 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=30336 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.708207065e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=935477248 dispatch_meta_bytes=870912 combine_payload_bytes=935477248 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=936348160 ingress_bytes=1463959552 link_bytes=1463959552 token_bytes=1871825408 weight_movement_bytes=528482304 total_nvlink_bytes=2400307712 weight_to_token_ratio=0.282335 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=29824 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.197562404e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=931889152 dispatch_meta_bytes=884256 combine_payload_bytes=931889152 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=932773408 ingress_bytes=1108049920 link_bytes=1108049920 token_bytes=1864662560 weight_movement_bytes=176160768 total_nvlink_bytes=2040823328 weight_to_token_ratio=0.094473 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.204617029e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=903424 combine_payload_bytes=939311104 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940214528 ingress_bytes=1291632640 link_bytes=1291632640 token_bytes=1879525632 weight_movement_bytes=352321536 total_nvlink_bytes=2231847168 weight_to_token_ratio=0.187452 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017045 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232603630e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131000 payload_rows_remote=114684 dup_rows=131144 dup_rows_remote=116296 dedup_ratio=0.500275 router_tokens_per_expert=[1, 0, 1, 6, 0, 0, 51, 0, 0, 0, 8184, 8188, 1348, 0, 1, 8191, 0, 1, 8190, 1, 1, 8183, 1, 8054, 0, 7688, 7893, 0, 8172, 1, 0, 0, 1359, 8190, 2, 1, 63, 8179, 8183, 1, 1, 163, 0, 0, 1, 6189, 0, 2, 1, 8190, 0, 1, 0, 4, 1, 8191, 0, 1, 8190, 0, 0, 1, 1, 1] payload_rows_per_dst=[16384, 16382, 16384, 16316, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16480, 19060, 16112, 14848, 19170, 12710, 16392, 16372] combine_rows_per_src=[16384, 16382, 16384, 16316, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131008 payload_rows_remote=114624 dup_rows=131136 dup_rows_remote=114748 dedup_ratio=0.500244 router_tokens_per_expert=[0, 1, 0, 25, 0, 0, 18, 0, 0, 0, 8190, 8189, 888, 0, 0, 8192, 0, 0, 8147, 0, 0, 8186, 0, 8041, 16, 6673, 7899, 0, 8185, 0, 0, 0, 2503, 8192, 2, 0, 6, 8080, 8183, 0, 0, 213, 0, 0, 0, 6664, 0, 1, 0, 8192, 0, 0, 0, 2, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16324, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16454, 2524, 15996, 12852, 21168, 29382, 16388, 16372] combine_rows_per_src=[16384, 16384, 16380, 16324, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 2, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130992 payload_rows_remote=114628 dup_rows=131152 dup_rows_remote=114892 dedup_ratio=0.500305 router_tokens_per_expert=[0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 8189, 8185, 1415, 0, 0, 8192, 0, 0, 8188, 0, 0, 8178, 0, 8134, 24, 6069, 7280, 0, 8191, 0, 0, 0, 3935, 8190, 1, 0, 2, 8142, 8186, 0, 0, 230, 0, 0, 0, 5744, 0, 12, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16364, 16360, 16348, 16384, 16384, 16384] dup_rows_per_dst=[16384, 19194, 16260, 17168, 17412, 11972, 16384, 16378] combine_rows_per_src=[16384, 16384, 16364, 16360, 16348, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114660 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 8191, 8147, 1091, 0, 0, 8192, 0, 0, 8160, 0, 0, 8189, 3, 8017, 6, 7576, 7806, 0, 8188, 0, 0, 0, 3046, 8183, 0, 0, 8, 8052, 8178, 0, 0, 112, 0, 0, 0, 5323, 0, 14, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16380, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16172, 2094, 15936, 14396, 22398, 27280, 16384, 16422] combine_rows_per_src=[16384, 16382, 16380, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=923920 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940415248 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=1879906576 weight_movement_bytes=704643072 total_nvlink_bytes=2584549648 weight_to_token_ratio=0.374829 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=28928 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114666 dup_rows=131096 dup_rows_remote=111666 dedup_ratio=0.500092 router_tokens_per_expert=[1, 0, 0, 25, 0, 0, 9, 0, 0, 0, 8182, 8175, 1056, 0, 1, 8191, 0, 1, 8190, 1, 1, 8172, 5, 7894, 0, 7590, 7918, 0, 8166, 1, 0, 0, 2710, 8190, 0, 5, 30, 8173, 8182, 1, 1, 278, 0, 0, 1, 5332, 0, 7, 1, 8191, 0, 1, 0, 3, 1, 8191, 1, 1, 8189, 0, 2, 1, 0, 1] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16364, 16384, 16384, 16382] dup_rows_per_dst=[16430, 19430, 15788, 29982, 5474, 11236, 16392, 16364] combine_rows_per_src=[16384, 16382, 16384, 16384, 16364, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114678 dup_rows=131084 dup_rows_remote=117460 dedup_ratio=0.500046 router_tokens_per_expert=[2, 2, 2, 13, 0, 1, 7, 0, 2, 0, 8186, 8167, 963, 0, 1, 8190, 0, 2, 8191, 1, 2, 8189, 1, 8069, 1, 7674, 7959, 0, 8175, 1, 0, 0, 1893, 8183, 1, 1, 13, 8151, 8189, 2, 1, 66, 0, 0, 0, 6161, 0, 23, 0, 8190, 2, 0, 2, 2, 1, 8189, 2, 0, 8189, 1, 7, 0, 0, 2] payload_rows_per_dst=[16384, 16380, 16384, 16380, 16384, 16382, 16382, 16384] dup_rows_per_dst=[16414, 17134, 16146, 14890, 20088, 13624, 16390, 16398] combine_rows_per_src=[16384, 16380, 16384, 16380, 16384, 16382, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=938999808 dispatch_meta_bytes=917488 combine_payload_bytes=938999808 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=939917296 ingress_bytes=1115160576 link_bytes=1115160576 token_bytes=1878917104 weight_movement_bytes=176160768 total_nvlink_bytes=2055077872 weight_to_token_ratio=0.093757 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244433735e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=918080 combine_payload_bytes=939032576 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939950656 ingress_bytes=1467514880 link_bytes=1467514880 token_bytes=1878983232 weight_movement_bytes=528482304 total_nvlink_bytes=2407465536 weight_to_token_ratio=0.281260 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217470757e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130984 payload_rows_remote=114600 dup_rows=131160 dup_rows_remote=114890 dedup_ratio=0.500336 router_tokens_per_expert=[1, 4, 1, 15, 0, 4, 13, 0, 0, 0, 8139, 8133, 2397, 0, 1, 8176, 7, 0, 8186, 1, 1, 8151, 15, 8103, 9, 6908, 7833, 0, 8151, 1, 1, 1, 1982, 8191, 0, 16, 41, 8141, 8098, 1, 1, 362, 0, 0, 1, 5349, 0, 42, 1, 8183, 0, 1, 0, 28, 1, 8174, 1, 1, 8184, 0, 20, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16348, 16382, 16336, 16384, 16384, 16382] dup_rows_per_dst=[16270, 33518, 18884, 14236, 4026, 11478, 16358, 16390] combine_rows_per_src=[16384, 16384, 16348, 16382, 16336, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917352 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940359528 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879801704 weight_movement_bytes=352321536 total_nvlink_bytes=2232123240 weight_to_token_ratio=0.187425 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=905328 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940249200 ingress_bytes=1467826176 link_bytes=1467826176 token_bytes=1879593072 weight_movement_bytes=528482304 total_nvlink_bytes=2408075376 weight_to_token_ratio=0.281168 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219213885e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=928552 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940370728 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879812904 weight_movement_bytes=352321536 total_nvlink_bytes=2232134440 weight_to_token_ratio=0.187424 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=109648 dedup_ratio=0.500061 router_tokens_per_expert=[0, 1, 0, 32, 0, 1, 20, 0, 0, 0, 8190, 8166, 900, 0, 0, 8192, 1, 0, 8191, 0, 0, 8191, 1, 8006, 2, 7396, 7225, 0, 8183, 0, 0, 0, 2553, 8192, 1, 1, 7, 8167, 8180, 0, 0, 42, 0, 0, 0, 6652, 0, 2, 0, 8192, 0, 0, 0, 1, 0, 8192, 0, 0, 8191, 0, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16368, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16468, 18128, 16014, 12878, 21440, 13392, 16386, 16382] combine_rows_per_src=[16384, 16384, 16384, 16368, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114234 dedup_ratio=0.500038 router_tokens_per_expert=[1, 1, 0, 0, 8181, 1, 1, 6591, 8188, 8118, 0, 1, 0, 2537, 1, 8189, 8187, 0, 8027, 8191, 1, 1, 1, 1429, 0, 0, 0, 0, 1, 0, 1, 237, 0, 1, 1, 8191, 25, 7789, 8187, 0, 8055, 8179, 1, 23, 0, 0, 0, 30, 8191, 0, 1, 5423, 42, 0, 4, 0, 8190, 25, 826, 1, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[13168, 21312, 18914, 16848, 15626, 16192, 10938, 18084] combine_rows_per_src=[16384, 16384, 16378, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=938803200 dispatch_meta_bytes=917960 combine_payload_bytes=938803200 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939721160 ingress_bytes=1291124736 link_bytes=1291124736 token_bytes=1878524360 weight_movement_bytes=352321536 total_nvlink_bytes=2230845896 weight_to_token_ratio=0.187552 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.229759580e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=897280 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940290304 ingress_bytes=1467875328 link_bytes=1467875328 token_bytes=1879683328 weight_movement_bytes=528482304 total_nvlink_bytes=2408165632 weight_to_token_ratio=0.281155 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.017992 gemm_active_groups=18 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219489115e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130834 payload_rows_remote=114450 dup_rows=131310 dup_rows_remote=107432 dedup_ratio=0.500908 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 4649, 8144, 8157, 0, 0, 0, 3266, 0, 8192, 8192, 0, 7781, 8192, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 8192, 39, 8047, 8190, 0, 8107, 8176, 0, 9, 0, 0, 0, 20, 8192, 0, 0, 5597, 12, 0, 0, 0, 8191, 1, 2019, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16352, 16182, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[9298, 6494, 12446, 17136, 25392, 16244, 23878, 20422] combine_rows_per_src=[16384, 16352, 16182, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=915648 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940357824 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879800000 weight_movement_bytes=352321536 total_nvlink_bytes=2232121536 weight_to_token_ratio=0.187425 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130938 payload_rows_remote=114608 dup_rows=131206 dup_rows_remote=115820 dedup_ratio=0.500511 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 6912, 8183, 7881, 0, 0, 0, 2691, 0, 8192, 8192, 0, 7784, 8192, 0, 0, 0, 1349, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 8192, 58, 7240, 8192, 0, 8145, 8053, 0, 7, 0, 0, 0, 56, 8192, 0, 0, 5681, 5, 0, 0, 0, 8191, 2, 1287, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16330, 16384, 16318, 16370, 16384, 16384] dup_rows_per_dst=[21178, 21144, 15386, 16772, 10242, 16152, 11372, 18960] combine_rows_per_src=[16384, 16384, 16330, 16384, 16318, 16370, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126748 payload_rows_remote=110364 dup_rows=135396 dup_rows_remote=114072 dedup_ratio=0.516495 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 5294, 8139, 7910, 0, 0, 0, 2560, 0, 8192, 8192, 0, 7511, 8192, 0, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 8192, 3, 7815, 8192, 0, 8129, 8069, 0, 36, 0, 0, 0, 76, 8192, 0, 0, 6408, 15, 0, 1, 0, 8191, 0, 2471, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16166, 12278, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[10588, 4774, 7672, 17084, 27090, 17632, 29232, 21324] combine_rows_per_src=[16384, 16166, 12278, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114782 dedup_ratio=0.500015 router_tokens_per_expert=[2, 2, 0, 3, 8189, 1, 0, 5682, 8166, 8106, 0, 2, 0, 1808, 0, 8187, 8189, 0, 7828, 8190, 2, 2, 2, 1048, 0, 0, 0, 0, 0, 2, 1, 309, 1, 2, 4, 8189, 34, 7845, 8189, 0, 8106, 8182, 2, 21, 0, 0, 0, 27, 8190, 0, 2, 7141, 8, 2, 1, 1, 8185, 7, 1209, 0, 0, 0, 1, 2] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[11374, 12834, 17760, 16946, 22758, 16294, 14306, 18804] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130692 payload_rows_remote=114308 dup_rows=131452 dup_rows_remote=110200 dedup_ratio=0.501450 router_tokens_per_expert=[1, 1, 0, 0, 8190, 1, 0, 4329, 8106, 8079, 0, 1, 0, 2548, 0, 8189, 8185, 0, 7882, 8191, 1, 1, 1, 1295, 0, 0, 0, 0, 1, 0, 1, 648, 1, 1, 2, 8190, 3, 7875, 8187, 0, 8098, 7994, 1, 19, 1, 0, 0, 30, 8191, 2, 1, 7004, 27, 1, 6, 0, 8187, 17, 1581, 1, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16366, 16384, 16040, 16366, 16384, 16384] dup_rows_per_dst=[25022, 21252, 19686, 17506, 7916, 15920, 14080, 10070] combine_rows_per_src=[16384, 16384, 16366, 16384, 16040, 16366, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=937574400 dispatch_meta_bytes=887528 combine_payload_bytes=937574400 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938461928 ingress_bytes=1113735168 link_bytes=1113735168 token_bytes=1876036328 weight_movement_bytes=176160768 total_nvlink_bytes=2052197096 weight_to_token_ratio=0.093901 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.236452045e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 2, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 4, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938868736 dispatch_meta_bytes=921712 combine_payload_bytes=938868736 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=939790448 ingress_bytes=1643511808 link_bytes=1643511808 token_bytes=1878659184 weight_movement_bytes=704643072 total_nvlink_bytes=2583302256 weight_to_token_ratio=0.375078 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.202980089e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=904101888 dispatch_meta_bytes=897744 combine_payload_bytes=904101888 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=904999632 ingress_bytes=1080262656 link_bytes=1080262656 token_bytes=1809101520 weight_movement_bytes=176160768 total_nvlink_bytes=1985262288 weight_to_token_ratio=0.097375 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.049019950e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917872 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425584 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879933296 weight_movement_bytes=176160768 total_nvlink_bytes=2056094064 weight_to_token_ratio=0.093706 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130138 payload_rows_remote=113754 dup_rows=132006 dup_rows_remote=102106 dedup_ratio=0.503563 router_tokens_per_expert=[1, 1, 0, 0, 8190, 0, 0, 6762, 7997, 7953, 0, 1, 0, 1069, 0, 8191, 8183, 0, 8015, 8191, 1, 1, 1, 712, 0, 0, 0, 0, 1, 1, 1, 259, 8, 1, 1, 8191, 9, 7250, 8190, 0, 8085, 8059, 1, 41, 0, 0, 1, 99, 8191, 0, 1, 7327, 61, 0, 19, 1, 8190, 0, 1808, 1, 2, 0, 3, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 15466, 16368, 16384, 16384] dup_rows_per_dst=[29900, 22834, 25640, 11726, 7260, 16204, 14816, 3626] combine_rows_per_src=[16384, 16384, 16384, 16384, 15466, 16368, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=936411136 dispatch_meta_bytes=898032 combine_payload_bytes=936411136 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=937309168 ingress_bytes=1464893440 link_bytes=1464893440 token_bytes=1873720304 weight_movement_bytes=528482304 total_nvlink_bytes=2402202608 weight_to_token_ratio=0.282050 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.202791787e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=114952 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 8190, 0, 0, 5274, 8149, 8064, 0, 0, 0, 2276, 0, 8192, 8190, 0, 8013, 8192, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 8191, 68, 7993, 8191, 0, 8162, 8187, 0, 39, 0, 0, 0, 48, 8192, 0, 0, 6164, 60, 0, 1, 0, 8191, 7, 1693, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[10544, 20678, 17956, 17054, 16122, 16488, 12450, 19782] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114676 dup_rows=131086 dup_rows_remote=114176 dedup_ratio=0.500053 router_tokens_per_expert=[0, 8191, 0, 0, 8150, 0, 1, 1, 8184, 8162, 2, 1, 1, 1, 0, 4219, 1, 1, 0, 13, 0, 1, 4030, 0, 8190, 1, 0, 265, 0, 8190, 0, 0, 20, 1, 8077, 0, 2, 8191, 0, 1, 8189, 8189, 8190, 1, 0, 1, 1, 202, 3, 8191, 1, 0, 0, 0, 9, 0, 1, 1, 8191, 8188, 1, 1, 7814, 1] payload_rows_per_dst=[16384, 16376, 16384, 16382, 16384, 16380, 16384, 16384] dup_rows_per_dst=[16302, 22680, 21708, 16910, 16200, 5250, 16406, 15630] combine_rows_per_src=[16384, 16376, 16384, 16382, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=931872768 dispatch_meta_bytes=863440 combine_payload_bytes=931872768 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=932736208 ingress_bytes=1108033536 link_bytes=1108033536 token_bytes=1864608976 weight_movement_bytes=176160768 total_nvlink_bytes=2040769744 weight_to_token_ratio=0.094476 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.204525286e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114680 dedup_ratio=0.500046 router_tokens_per_expert=[0, 8192, 0, 1, 8093, 0, 0, 0, 8183, 8184, 2, 0, 0, 0, 0, 4072, 0, 0, 0, 5, 0, 0, 4559, 0, 8192, 0, 0, 97, 0, 8192, 0, 4, 1, 0, 8043, 0, 3, 8192, 0, 0, 8191, 8189, 8192, 0, 0, 0, 0, 123, 3, 8192, 0, 0, 0, 0, 7, 0, 1, 0, 8192, 8187, 0, 0, 7780, 0] payload_rows_per_dst=[16384, 16374, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16188, 8140, 9126, 16586, 16094, 32992, 16404, 15554] combine_rows_per_src=[16384, 16374, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=918560 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940442656 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879966752 weight_movement_bytes=528482304 total_nvlink_bytes=2408449056 weight_to_token_ratio=0.281113 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=915408 combine_payload_bytes=939425792 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940341200 ingress_bytes=1379827712 link_bytes=1379827712 token_bytes=1879766992 weight_movement_bytes=440401920 total_nvlink_bytes=2320168912 weight_to_token_ratio=0.234285 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726459219e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128580 payload_rows_remote=112196 dup_rows=133564 dup_rows_remote=110836 dedup_ratio=0.509506 router_tokens_per_expert=[0, 8192, 0, 1, 7908, 0, 0, 0, 8152, 8188, 15, 0, 0, 0, 0, 4568, 0, 0, 0, 2, 0, 0, 4790, 0, 8192, 0, 0, 71, 0, 8192, 0, 1, 14, 0, 8144, 0, 0, 8192, 0, 0, 8192, 8189, 8192, 0, 0, 0, 0, 102, 9, 8192, 0, 0, 0, 0, 7, 0, 21, 0, 8192, 8185, 0, 0, 7169, 0] payload_rows_per_dst=[16384, 13892, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15818, 3582, 9584, 16422, 16762, 32958, 15710, 22728] combine_rows_per_src=[16384, 13892, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=107750 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 8127, 0, 0, 0, 8155, 8190, 2, 0, 0, 0, 0, 5486, 1, 0, 0, 0, 0, 0, 3469, 0, 8192, 0, 0, 3, 0, 8192, 0, 1, 12, 0, 8149, 0, 0, 8192, 0, 0, 8192, 8192, 8191, 0, 0, 0, 0, 44, 0, 8192, 0, 0, 0, 0, 21, 0, 1, 0, 8192, 8192, 0, 0, 7492, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16254, 27282, 23322, 16392, 16322, 88, 16426, 14986] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=98232 dedup_ratio=0.500038 router_tokens_per_expert=[3, 8190, 0, 1, 8111, 0, 2, 2, 8179, 8180, 10, 2, 2, 2, 0, 3532, 0, 1, 0, 9, 0, 2, 4797, 0, 8189, 3, 1, 90, 0, 8189, 0, 0, 29, 2, 8175, 0, 3, 8190, 0, 0, 8189, 8181, 8182, 2, 0, 2, 0, 70, 69, 8190, 2, 0, 0, 0, 18, 2, 1, 2, 8189, 8182, 2, 1, 7890, 2] payload_rows_per_dst=[16384, 16378, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16234, 7076, 9612, 16560, 16418, 32850, 16556, 15776] combine_rows_per_src=[16384, 16378, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114638 dup_rows=131122 dup_rows_remote=106786 dedup_ratio=0.500191 router_tokens_per_expert=[0, 8189, 0, 1, 8185, 0, 1, 1, 8143, 8157, 37, 1, 1, 1, 0, 4020, 3, 1, 0, 6, 0, 1, 4374, 0, 8191, 0, 4, 131, 0, 8189, 0, 0, 42, 6, 8168, 0, 0, 8191, 0, 1, 8191, 8167, 8188, 1, 0, 1, 1, 70, 0, 8191, 1, 0, 0, 0, 30, 0, 1, 1, 8191, 8179, 1, 1, 7811, 1] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16336, 16384, 16384] dup_rows_per_dst=[16370, 24336, 25144, 16648, 16432, 144, 16442, 15606] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16336, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917424 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940343216 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879769008 weight_movement_bytes=176160768 total_nvlink_bytes=2055929776 weight_to_token_ratio=0.093714 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=919109632 dispatch_meta_bytes=892128 combine_payload_bytes=919109632 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=920001760 ingress_bytes=1271431168 link_bytes=1271431168 token_bytes=1839111392 weight_movement_bytes=352321536 total_nvlink_bytes=2191432928 weight_to_token_ratio=0.191572 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.119483819e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=889752 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940413848 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879937944 weight_movement_bytes=352321536 total_nvlink_bytes=2232259480 weight_to_token_ratio=0.187411 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=851640 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940293816 ingress_bytes=1467924480 link_bytes=1467924480 token_bytes=1879735992 weight_movement_bytes=528482304 total_nvlink_bytes=2408218296 weight_to_token_ratio=0.281147 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219764346e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=885696 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940000192 ingress_bytes=1555677184 link_bytes=1555677184 token_bytes=1879114688 weight_movement_bytes=616562688 total_nvlink_bytes=2495677376 weight_to_token_ratio=0.328113 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=31360 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.711142859e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130950 payload_rows_remote=114566 dup_rows=131194 dup_rows_remote=113674 dedup_ratio=0.500465 router_tokens_per_expert=[1, 8182, 0, 1, 8149, 0, 1, 1, 8036, 8136, 64, 1, 1, 1, 0, 5212, 12, 1, 0, 0, 0, 1, 3419, 0, 8190, 0, 8, 49, 1, 8179, 0, 1, 72, 27, 8186, 3, 1, 8191, 0, 1, 8190, 8138, 8186, 1, 0, 1, 1, 34, 12, 8191, 1, 0, 0, 0, 122, 0, 40, 1, 8186, 8149, 1, 1, 7688, 1] payload_rows_per_dst=[16384, 16368, 16384, 16382, 16384, 16280, 16384, 16384] dup_rows_per_dst=[17520, 26914, 23234, 14906, 16532, 70, 16640, 15378] combine_rows_per_src=[16384, 16368, 16384, 16382, 16384, 16280, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=121468 dedup_ratio=0.500069 router_tokens_per_expert=[0, 1, 8191, 8188, 0, 1, 0, 0, 0, 0, 0, 1, 3616, 1, 13, 6806, 8190, 1, 1, 0, 2, 0, 1, 0, 8191, 1, 8191, 0, 1, 1, 1, 8183, 4554, 0, 0, 1, 3, 8191, 8176, 8191, 1, 1, 0, 8190, 8190, 1, 8191, 4, 0, 1, 0, 3369, 0, 0, 7765, 1, 0, 1, 1733, 4914, 0, 10, 0, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16368, 16384] dup_rows_per_dst=[16378, 20872, 16388, 9622, 17276, 16390, 20846, 13318] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16368, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114758 dedup_ratio=0.500008 router_tokens_per_expert=[0, 8192, 1, 1, 8191, 0, 0, 0, 8161, 8187, 30, 0, 0, 0, 0, 5303, 2, 0, 0, 4, 0, 0, 3098, 0, 8191, 0, 1, 135, 0, 8191, 0, 1, 2, 4, 8151, 0, 1, 8192, 0, 0, 8192, 8178, 8191, 0, 0, 0, 0, 136, 4, 8192, 0, 0, 0, 0, 49, 0, 15, 0, 8191, 8179, 0, 0, 7706, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16386, 13596, 9342, 16654, 16316, 26876, 16488, 15416] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=938524672 dispatch_meta_bytes=912960 combine_payload_bytes=938524672 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939437632 ingress_bytes=1114685440 link_bytes=1114685440 token_bytes=1877962304 weight_movement_bytes=176160768 total_nvlink_bytes=2054123072 weight_to_token_ratio=0.093804 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.241773171e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=944552 combine_payload_bytes=939376640 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940321192 ingress_bytes=1467858944 link_bytes=1467858944 token_bytes=1879697832 weight_movement_bytes=528482304 total_nvlink_bytes=2408180136 weight_to_token_ratio=0.281153 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219397372e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129890 payload_rows_remote=113506 dup_rows=132254 dup_rows_remote=110972 dedup_ratio=0.504509 router_tokens_per_expert=[0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 0, 0, 6677, 8192, 0, 0, 0, 40, 0, 0, 0, 8192, 0, 8190, 0, 0, 0, 0, 8189, 4095, 0, 3, 0, 1, 8192, 8189, 8192, 0, 0, 0, 8192, 8191, 0, 8192, 0, 0, 0, 0, 3447, 0, 0, 7194, 0, 0, 0, 2603, 6036, 0, 6, 0, 0] payload_rows_per_dst=[16384, 15204, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 8688, 16464, 16024, 19740, 16382, 21282, 17290] combine_rows_per_src=[16384, 15204, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917776 combine_payload_bytes=939507712 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940425488 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879933200 weight_movement_bytes=176160768 total_nvlink_bytes=2056093968 weight_to_token_ratio=0.093706 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130278 payload_rows_remote=113894 dup_rows=131866 dup_rows_remote=106296 dedup_ratio=0.503029 router_tokens_per_expert=[0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 0, 7221, 8192, 0, 0, 0, 74, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 8189, 3995, 0, 0, 0, 1, 8192, 8189, 8192, 0, 0, 0, 8192, 8190, 0, 8192, 0, 0, 0, 0, 3091, 0, 0, 7877, 0, 2, 0, 2292, 5457, 0, 51, 0, 3] payload_rows_per_dst=[16384, 15590, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 4282, 15516, 16378, 14410, 17396, 21934, 25570] combine_rows_per_src=[16384, 15590, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114286 dedup_ratio=0.500053 router_tokens_per_expert=[0, 0, 8192, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 4389, 0, 0, 7670, 8192, 0, 0, 0, 208, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 2639, 0, 1, 0, 1, 8192, 8192, 8192, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 1897, 0, 0, 8131, 0, 0, 0, 2799, 5025, 0, 20, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16370, 16384] dup_rows_per_dst=[16360, 24118, 16800, 18332, 13424, 16384, 9980, 15688] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130920 payload_rows_remote=114536 dup_rows=131224 dup_rows_remote=111966 dedup_ratio=0.500580 router_tokens_per_expert=[0, 1, 8191, 8191, 0, 1, 0, 0, 0, 0, 0, 1, 2235, 1, 1, 7392, 8191, 1, 1, 0, 102, 0, 1, 1, 8191, 0, 8191, 0, 1, 0, 1, 8178, 4346, 0, 0, 1, 7, 8191, 8178, 8190, 1, 1, 0, 8190, 8190, 1, 8191, 6, 0, 1, 0, 2569, 1, 0, 8044, 1, 1, 1, 1276, 6771, 0, 38, 0, 3] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16234, 16384] dup_rows_per_dst=[16384, 19258, 16592, 24546, 16874, 16394, 4998, 16178] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16234, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114644 dup_rows=131116 dup_rows_remote=114732 dedup_ratio=0.500168 router_tokens_per_expert=[0, 1, 8190, 8185, 0, 2, 0, 2, 1, 0, 0, 2, 2948, 0, 12, 7432, 8190, 2, 0, 0, 26, 1, 1, 4, 8190, 2, 8190, 1, 2, 0, 1, 8186, 3451, 1, 2, 1, 6, 8189, 8185, 8189, 2, 0, 1, 8190, 8189, 2, 8190, 0, 3, 2, 0, 4371, 0, 0, 7906, 2, 1, 2, 1642, 4960, 0, 11, 0, 3] payload_rows_per_dst=[16384, 16342, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16376, 15954, 16444, 13068, 15094, 16384, 24564, 13232] combine_rows_per_src=[16384, 16342, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=929841152 dispatch_meta_bytes=897912 combine_payload_bytes=929841152 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=930739064 ingress_bytes=1106001920 link_bytes=1106001920 token_bytes=1860580216 weight_movement_bytes=176160768 total_nvlink_bytes=2036740984 weight_to_token_ratio=0.094681 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=31872 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.193149084e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=933019648 dispatch_meta_bytes=880760 combine_payload_bytes=933019648 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=933900408 ingress_bytes=1285341184 link_bytes=1285341184 token_bytes=1866920056 weight_movement_bytes=352321536 total_nvlink_bytes=2219241592 weight_to_token_ratio=0.188718 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.197374103e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938278912 dispatch_meta_bytes=906008 combine_payload_bytes=938278912 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939184920 ingress_bytes=1114439680 link_bytes=1114439680 token_bytes=1877463832 weight_movement_bytes=176160768 total_nvlink_bytes=2053624600 weight_to_token_ratio=0.093829 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=29056 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.240397018e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=915840 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940325248 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879734656 weight_movement_bytes=176160768 total_nvlink_bytes=2055895424 weight_to_token_ratio=0.093716 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130764 payload_rows_remote=114380 dup_rows=131380 dup_rows_remote=114952 dedup_ratio=0.501175 router_tokens_per_expert=[0, 1, 8191, 8188, 0, 1, 0, 0, 0, 0, 0, 1, 2835, 0, 39, 7238, 8171, 1, 1, 0, 52, 0, 13, 0, 8191, 1, 8191, 0, 1, 0, 1, 8150, 3154, 0, 2, 1, 0, 8191, 8163, 8190, 1, 0, 1, 8191, 8191, 1, 8191, 2, 1, 1, 0, 3573, 1, 0, 7887, 1, 0, 1, 1604, 6400, 0, 32, 0, 25] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16340, 16120, 16384] dup_rows_per_dst=[16428, 20224, 16474, 24920, 14998, 15406, 6808, 16122] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16340, 16120, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=917504 combine_payload_bytes=939163648 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=940081152 ingress_bytes=1467645952 link_bytes=1467645952 token_bytes=1879244800 weight_movement_bytes=528482304 total_nvlink_bytes=2407727104 weight_to_token_ratio=0.281221 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218204706e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114644 dup_rows=131116 dup_rows_remote=112068 dedup_ratio=0.500168 router_tokens_per_expert=[1, 0, 4, 1, 6345, 0, 0, 0, 8187, 0, 8191, 8186, 1, 401, 0, 1, 8107, 0, 1, 7501, 8190, 0, 8191, 0, 0, 1333, 8191, 0, 1, 8190, 1, 0, 1, 0, 67, 1, 1, 130, 0, 0, 1, 8190, 1, 3, 0, 0, 688, 1, 8191, 8191, 1, 1, 0, 8190, 4, 0, 0, 1, 1, 1, 1, 8191, 0, 8191] payload_rows_per_dst=[16384, 16378, 16346, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[12700, 17174, 14872, 19048, 16778, 17766, 16390, 16388] combine_rows_per_src=[16384, 16378, 16346, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=111816 dedup_ratio=0.500015 router_tokens_per_expert=[0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2467, 0, 2, 7377, 8192, 0, 0, 0, 3, 0, 1, 1, 8192, 0, 8192, 0, 0, 0, 0, 8189, 5537, 0, 0, 0, 2, 8192, 8187, 8192, 0, 0, 0, 8191, 8191, 0, 8192, 0, 0, 0, 0, 2535, 0, 0, 8017, 0, 0, 0, 1390, 5394, 0, 53, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 19692, 16394, 8190, 19260, 16380, 21104, 13674] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=937000960 dispatch_meta_bytes=917328 combine_payload_bytes=937000960 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=937918288 ingress_bytes=1113161728 link_bytes=1113161728 token_bytes=1874919248 weight_movement_bytes=176160768 total_nvlink_bytes=2051080016 weight_to_token_ratio=0.093956 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.233241020e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=906000 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940397328 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=1879888656 weight_movement_bytes=704643072 total_nvlink_bytes=2584531728 weight_to_token_ratio=0.374832 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.017992 gemm_active_groups=17 gemm_peak_group_rows=31616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=906848 combine_payload_bytes=939163648 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940070496 ingress_bytes=1467645952 link_bytes=1467645952 token_bytes=1879234144 weight_movement_bytes=528482304 total_nvlink_bytes=2407716448 weight_to_token_ratio=0.281222 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.021780 gemm_active_groups=14 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218204706e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=112934 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 0, 6240, 0, 0, 0, 8181, 0, 8192, 8192, 0, 112, 0, 0, 8172, 0, 0, 6555, 8189, 0, 8192, 0, 0, 1523, 8192, 0, 0, 8192, 0, 0, 0, 0, 17, 0, 0, 537, 0, 0, 0, 8192, 0, 0, 0, 0, 1434, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 8192] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[12480, 16586, 13072, 19430, 17486, 17496, 18140, 16384] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131012 payload_rows_remote=114628 dup_rows=131132 dup_rows_remote=118140 dedup_ratio=0.500229 router_tokens_per_expert=[1, 0, 5, 1, 5905, 0, 1, 0, 8173, 1, 8190, 8184, 0, 328, 4, 1, 8055, 0, 1, 7776, 8185, 0, 8191, 0, 0, 879, 8189, 0, 1, 8191, 1, 0, 1, 0, 94, 1, 1, 90, 0, 0, 1, 8191, 4, 1, 0, 1, 1460, 1, 8190, 8191, 1, 1, 0, 8189, 3, 1, 0, 1, 1, 1, 4, 8189, 0, 8191] payload_rows_per_dst=[16384, 16384, 16326, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[22664, 12992, 15338, 18138, 16742, 19316, 9552, 16390] combine_rows_per_src=[16384, 16384, 16326, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=114714 dedup_ratio=0.500099 router_tokens_per_expert=[0, 0, 0, 0, 6923, 0, 0, 0, 8182, 0, 8192, 8192, 0, 191, 0, 0, 8059, 0, 0, 7441, 8189, 0, 8192, 0, 0, 564, 8191, 0, 0, 8192, 0, 0, 0, 0, 40, 0, 0, 182, 0, 0, 0, 8192, 1, 0, 0, 0, 1188, 0, 8192, 8192, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16358, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[13846, 16746, 14642, 12442, 16822, 2378, 37838, 16384] combine_rows_per_src=[16384, 16384, 16358, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114688 dup_rows=131082 dup_rows_remote=115888 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 0, 6248, 0, 0, 0, 8179, 1, 8192, 8192, 0, 358, 0, 0, 8156, 0, 0, 7628, 8192, 0, 8192, 0, 0, 954, 8192, 0, 0, 8192, 0, 0, 0, 0, 45, 0, 0, 338, 0, 0, 0, 8192, 0, 0, 0, 0, 861, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[12496, 17076, 15194, 18292, 17150, 18106, 16384, 16384] combine_rows_per_src=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131050 payload_rows_remote=114666 dup_rows=131094 dup_rows_remote=112396 dedup_ratio=0.500084 router_tokens_per_expert=[2, 0, 14, 2, 6965, 1, 0, 2, 8171, 3, 8188, 8187, 2, 203, 1, 2, 8117, 0, 2, 7407, 8189, 0, 8185, 0, 0, 488, 8189, 0, 2, 8186, 2, 0, 2, 0, 127, 3, 2, 125, 1, 0, 2, 8189, 1, 0, 2, 0, 1155, 2, 8190, 8190, 3, 1, 0, 8188, 1, 2, 0, 2, 2, 0, 1, 8184, 0, 8187] payload_rows_per_dst=[16384, 16384, 16372, 16382, 16384, 16384, 16382, 16378] dup_rows_per_dst=[13964, 16754, 14680, 17352, 16884, 18698, 16388, 16374] combine_rows_per_src=[16384, 16384, 16372, 16382, 16384, 16384, 16382, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=910480 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940418192 ingress_bytes=1467990016 link_bytes=1467990016 token_bytes=1879925904 weight_movement_bytes=528482304 total_nvlink_bytes=2408408208 weight_to_token_ratio=0.281119 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220131320e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=931072 combine_payload_bytes=939032576 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939963648 ingress_bytes=1467514880 link_bytes=1467514880 token_bytes=1878996224 weight_movement_bytes=528482304 total_nvlink_bytes=2407478528 weight_to_token_ratio=0.281258 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.019886 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217470757e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=100714 dedup_ratio=0.500008 router_tokens_per_expert=[1, 0, 1, 1, 6991, 0, 9, 0, 8157, 0, 8191, 8178, 0, 374, 12, 1, 7979, 0, 1, 7625, 8179, 0, 8191, 0, 0, 477, 8179, 0, 1, 8184, 1, 0, 1, 0, 155, 1, 1, 493, 0, 0, 1, 8191, 10, 0, 0, 1, 510, 1, 8190, 8191, 1, 1, 0, 8191, 0, 0, 0, 1, 1, 0, 16, 8191, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[30360, 17086, 29154, 8352, 12316, 17426, 0, 16380] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=917504 combine_payload_bytes=939311104 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940228608 ingress_bytes=1115471872 link_bytes=1115471872 token_bytes=1879539712 weight_movement_bytes=176160768 total_nvlink_bytes=2055700480 weight_to_token_ratio=0.093725 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246176862e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=922304 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940446400 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879970496 weight_movement_bytes=528482304 total_nvlink_bytes=2408452800 weight_to_token_ratio=0.281112 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.025568 gemm_active_groups=12 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=114204 dedup_ratio=0.500099 router_tokens_per_expert=[0, 0, 0, 0, 6281, 0, 0, 0, 8171, 0, 8192, 8188, 0, 164, 1, 0, 8026, 0, 0, 7299, 8191, 0, 8192, 0, 0, 820, 8192, 0, 0, 8191, 0, 0, 0, 0, 27, 0, 0, 229, 0, 0, 0, 8192, 1, 0, 0, 0, 1750, 0, 8192, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 8192] payload_rows_per_dst=[16384, 16380, 16362, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[12562, 16668, 14288, 18022, 16894, 19886, 16384, 16394] combine_rows_per_src=[16384, 16380, 16362, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114638 dup_rows=131122 dup_rows_remote=118686 dedup_ratio=0.500191 router_tokens_per_expert=[0, 0, 0, 8191, 3406, 1, 0, 8191, 1, 8134, 1, 1, 8157, 4, 1, 8173, 8191, 0, 0, 8190, 1, 0, 8169, 8191, 413, 0, 155, 18, 1, 8191, 5632, 0, 7197, 2, 7259, 0, 1, 0, 0, 0, 0, 0, 615, 0, 1, 1, 0, 0, 8190, 1, 1, 0, 0, 1, 0, 1, 0, 8189, 8191, 1, 7, 0, 0, 1] payload_rows_per_dst=[16384, 16362, 16382, 16384, 16360, 16384, 16382, 16384] dup_rows_per_dst=[23194, 16236, 16338, 12436, 12558, 17614, 16352, 16394] combine_rows_per_src=[16384, 16362, 16382, 16384, 16360, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=908248 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940252120 ingress_bytes=1115504640 link_bytes=1115504640 token_bytes=1879595992 weight_movement_bytes=176160768 total_nvlink_bytes=2055756760 weight_to_token_ratio=0.093723 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246360349e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=861600 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940369312 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879877024 weight_movement_bytes=352321536 total_nvlink_bytes=2232198560 weight_to_token_ratio=0.187417 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=915464 combine_payload_bytes=939311104 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940226568 ingress_bytes=1291632640 link_bytes=1291632640 token_bytes=1879537672 weight_movement_bytes=352321536 total_nvlink_bytes=2231859208 weight_to_token_ratio=0.187451 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232603630e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=933296 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940047792 ingress_bytes=1115275264 link_bytes=1115275264 token_bytes=1879162288 weight_movement_bytes=176160768 total_nvlink_bytes=2055323056 weight_to_token_ratio=0.093744 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.022727 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245075940e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123672 payload_rows_remote=107288 dup_rows=138472 dup_rows_remote=122088 dedup_ratio=0.528229 router_tokens_per_expert=[0, 0, 0, 8192, 2492, 0, 0, 8192, 0, 8177, 0, 0, 8149, 2, 0, 8192, 8192, 0, 0, 8192, 0, 0, 8173, 8192, 672, 0, 145, 10, 0, 8192, 5577, 0, 7492, 0, 7933, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[8988, 16382, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[4396, 16274, 16346, 24606, 27038, 17044, 16384, 16384] combine_rows_per_src=[8988, 16382, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131016 payload_rows_remote=114654 dup_rows=131128 dup_rows_remote=114906 dedup_ratio=0.500214 router_tokens_per_expert=[0, 0, 0, 8191, 5002, 1, 0, 8191, 1, 8154, 1, 1, 8122, 12, 1, 8176, 8190, 0, 0, 8186, 1, 0, 8096, 8191, 634, 4, 16, 33, 1, 8191, 5044, 0, 5926, 13, 7696, 0, 1, 0, 0, 0, 0, 0, 417, 0, 1, 1, 0, 0, 8189, 1, 1, 0, 0, 1, 0, 1, 0, 8191, 8191, 1, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16362, 16374, 16384, 16364, 16384, 16380, 16384] dup_rows_per_dst=[26386, 16222, 16192, 11462, 10908, 17216, 16358, 16384] combine_rows_per_src=[16384, 16362, 16374, 16384, 16364, 16384, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125290 payload_rows_remote=108906 dup_rows=136854 dup_rows_remote=120470 dedup_ratio=0.522057 router_tokens_per_expert=[0, 0, 0, 8192, 5302, 0, 0, 8192, 0, 8182, 0, 0, 8059, 24, 0, 8192, 8192, 0, 0, 8192, 0, 0, 8126, 8192, 915, 1, 6, 19, 0, 8192, 4851, 0, 6780, 0, 6588, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[10604, 16382, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 15984, 16252, 27968, 26736, 16982, 16548, 16384] combine_rows_per_src=[10604, 16382, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130834 payload_rows_remote=114450 dup_rows=131310 dup_rows_remote=115204 dedup_ratio=0.500908 router_tokens_per_expert=[0, 0, 0, 8192, 5461, 0, 0, 8192, 0, 8188, 0, 0, 8150, 6, 0, 8192, 8192, 0, 0, 8192, 0, 0, 8053, 8192, 706, 0, 2, 6, 0, 8192, 6594, 0, 5054, 0, 6867, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 12, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16148, 16384, 16384, 16384] dup_rows_per_dst=[27306, 16306, 16106, 14616, 7694, 16490, 16384, 16408] combine_rows_per_src=[16384, 16382, 16384, 16384, 16148, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114260 dedup_ratio=0.500046 router_tokens_per_expert=[3, 0, 0, 8189, 2490, 2, 0, 8190, 1, 8178, 2, 2, 8157, 5, 2, 8161, 8189, 0, 3, 8189, 2, 0, 8164, 8189, 351, 0, 29, 9, 2, 8190, 6161, 0, 7646, 4, 7737, 0, 2, 0, 0, 1, 0, 0, 222, 0, 2, 2, 0, 0, 8189, 1, 2, 0, 1, 2, 0, 2, 0, 8188, 8190, 0, 20, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16380, 16384, 16382, 16380] dup_rows_per_dst=[14194, 16310, 16334, 13100, 21570, 16824, 16334, 16418] combine_rows_per_src=[16384, 16384, 16382, 16384, 16380, 16384, 16382, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[2, 2, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=878903296 dispatch_meta_bytes=917504 combine_payload_bytes=878903296 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=879820800 ingress_bytes=1231224832 link_bytes=1231224832 token_bytes=1758724096 weight_movement_bytes=352321536 total_nvlink_bytes=2111045632 weight_to_token_ratio=0.200328 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.894345121e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=918240 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940163808 ingress_bytes=1467727872 link_bytes=1467727872 token_bytes=1879409376 weight_movement_bytes=528482304 total_nvlink_bytes=2407891680 weight_to_token_ratio=0.281196 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218663423e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=892157952 dispatch_meta_bytes=917504 combine_payload_bytes=892157952 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=893075456 ingress_bytes=1068318720 link_bytes=1068318720 token_bytes=1785233408 weight_movement_bytes=176160768 total_nvlink_bytes=1961394176 weight_to_token_ratio=0.098677 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.982138894e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114682 dup_rows=131080 dup_rows_remote=116234 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 0, 8192, 2030, 0, 0, 8192, 0, 8176, 0, 0, 8134, 16, 0, 8192, 8192, 0, 0, 8191, 0, 0, 8140, 8192, 1339, 1, 93, 36, 0, 8192, 5468, 0, 7536, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16382, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[20444, 16272, 16280, 13874, 14846, 16596, 16384, 16384] combine_rows_per_src=[16384, 16380, 16382, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130912 payload_rows_remote=114528 dup_rows=131232 dup_rows_remote=102904 dedup_ratio=0.500610 router_tokens_per_expert=[0, 0, 0, 8191, 4459, 1, 0, 8191, 1, 8103, 1, 1, 7986, 43, 1, 8166, 8191, 0, 0, 8172, 1, 0, 8046, 8187, 843, 10, 22, 50, 1, 8191, 5580, 0, 5763, 0, 7426, 0, 1, 0, 0, 7, 5, 0, 817, 0, 1, 1, 0, 0, 8190, 1, 1, 0, 0, 1, 0, 1, 0, 8191, 8191, 1, 34, 1, 0, 1] payload_rows_per_dst=[16384, 16376, 16384, 16384, 16234, 16384, 16382, 16384] dup_rows_per_dst=[28328, 17530, 17240, 13010, 10160, 12306, 16338, 16320] combine_rows_per_src=[16384, 16376, 16384, 16384, 16234, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114284 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 8191, 1, 0, 1, 11, 0, 0, 1076, 8173, 0, 1, 1, 0, 63, 1, 13, 1, 657, 1, 0, 1, 0, 8184, 1, 1, 1, 211, 8190, 3, 1, 8191, 0, 0, 8190, 0, 0, 1, 77, 7594, 8158, 7923, 0, 8191, 8162, 0, 7945, 1, 1077, 10, 1, 3, 0, 8190, 1, 1, 8174, 6, 0, 6201, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16384] dup_rows_per_dst=[16340, 18500, 17796, 16798, 16382, 14814, 18070, 12382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=937574400 dispatch_meta_bytes=918616 combine_payload_bytes=937574400 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=938493016 ingress_bytes=1818378240 link_bytes=1818378240 token_bytes=1876067416 weight_movement_bytes=880803840 total_nvlink_bytes=2756871256 weight_to_token_ratio=0.469495 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32256 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.018215912e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=915744 combine_payload_bytes=939425792 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940341536 ingress_bytes=1291747328 link_bytes=1291747328 token_bytes=1879767328 weight_movement_bytes=352321536 total_nvlink_bytes=2232088864 weight_to_token_ratio=0.187428 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233245835e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=923664 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940398608 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879873552 weight_movement_bytes=176160768 total_nvlink_bytes=2056034320 weight_to_token_ratio=0.093709 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.021780 gemm_active_groups=13 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=938213376 dispatch_meta_bytes=869728 combine_payload_bytes=938213376 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939083104 ingress_bytes=1466695680 link_bytes=1466695680 token_bytes=1877296480 weight_movement_bytes=528482304 total_nvlink_bytes=2405778784 weight_to_token_ratio=0.281512 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.212883579e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=915848 combine_payload_bytes=939442176 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940358024 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879800200 weight_movement_bytes=352321536 total_nvlink_bytes=2232121736 weight_to_token_ratio=0.187425 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114640 dup_rows=131120 dup_rows_remote=114322 dedup_ratio=0.500183 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2912, 8192, 0, 0, 0, 0, 78, 0, 10, 0, 5, 0, 0, 0, 0, 8187, 0, 0, 0, 6, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 2, 7590, 8166, 7274, 0, 8192, 8092, 0, 8088, 0, 309, 0, 0, 2, 0, 8192, 0, 0, 8187, 2, 0, 6626, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16336, 16384, 16384] dup_rows_per_dst=[16332, 5824, 16370, 16386, 16384, 13396, 16798, 29630] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16336, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131000 payload_rows_remote=114616 dup_rows=131144 dup_rows_remote=110436 dedup_ratio=0.500275 router_tokens_per_expert=[0, 0, 0, 8191, 0, 0, 5, 7, 0, 0, 2168, 8185, 0, 1, 1, 0, 146, 1, 13, 1, 330, 1, 0, 1, 1, 8178, 1, 1, 1, 168, 8191, 3, 1, 8191, 1, 0, 8189, 0, 0, 1, 114, 7773, 8128, 7606, 0, 8190, 8050, 0, 7699, 1, 915, 4, 1, 0, 0, 8187, 1, 1, 8186, 5, 0, 6042, 0, 8191] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 16372, 16384, 16328] dup_rows_per_dst=[16280, 20708, 17084, 16704, 16382, 15394, 17230, 11362] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 16372, 16384, 16328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130926 payload_rows_remote=114542 dup_rows=131218 dup_rows_remote=96734 dedup_ratio=0.500557 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 2485, 8192, 0, 0, 0, 0, 295, 0, 10, 0, 25, 0, 0, 0, 0, 8188, 0, 0, 0, 4, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 7600, 8077, 7825, 0, 8192, 8105, 0, 7410, 0, 230, 2, 0, 11, 0, 8192, 0, 0, 8189, 2, 0, 7077, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16370, 16384, 16328, 16308, 16384] dup_rows_per_dst=[16156, 4970, 10624, 14840, 16400, 20768, 12976, 34484] combine_rows_per_src=[16384, 16384, 16384, 16370, 16384, 16328, 16308, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130962 payload_rows_remote=114578 dup_rows=131182 dup_rows_remote=114146 dedup_ratio=0.500420 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2288, 8192, 0, 0, 0, 0, 496, 0, 18, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 7206, 8146, 7852, 0, 8192, 8002, 0, 7961, 0, 136, 1, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 7043, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16274, 16384, 16384] dup_rows_per_dst=[16292, 20960, 17036, 16388, 16384, 13842, 16196, 14084] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16274, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114676 dup_rows=131100 dup_rows_remote=116478 dedup_ratio=0.500107 router_tokens_per_expert=[0, 0, 0, 8189, 0, 0, 2, 26, 0, 0, 1794, 8166, 0, 2, 2, 1, 25, 2, 16, 4, 1023, 2, 2, 2, 2, 8184, 2, 2, 1, 415, 8190, 6, 2, 8187, 0, 0, 8188, 0, 0, 2, 160, 7537, 8161, 7795, 2, 8185, 8151, 1, 7707, 2, 746, 14, 2, 4, 0, 8189, 4, 2, 8188, 3, 0, 5591, 0, 8189] payload_rows_per_dst=[16380, 16382, 16382, 16384, 16380, 16368, 16384, 16384] dup_rows_per_dst=[16376, 14062, 18442, 17220, 16378, 14622, 16944, 17056] combine_rows_per_src=[16380, 16382, 16382, 16384, 16380, 16368, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114650 dup_rows=131110 dup_rows_remote=114726 dedup_ratio=0.500145 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 3057, 8192, 0, 0, 0, 0, 154, 0, 19, 0, 1, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 7659, 8024, 7429, 0, 8192, 8136, 0, 7733, 0, 263, 0, 0, 10, 0, 8191, 0, 0, 8188, 7, 0, 6676, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16346, 16384, 16384] dup_rows_per_dst=[16050, 22498, 16620, 16360, 16384, 13830, 16010, 13358] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16346, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=915848 combine_payload_bytes=939130880 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940046728 ingress_bytes=1291452416 link_bytes=1291452416 token_bytes=1879177608 weight_movement_bytes=352321536 total_nvlink_bytes=2231499144 weight_to_token_ratio=0.187487 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.231594451e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938934272 dispatch_meta_bytes=900208 combine_payload_bytes=938934272 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939834480 ingress_bytes=1115095040 link_bytes=1115095040 token_bytes=1878768752 weight_movement_bytes=176160768 total_nvlink_bytes=2054929520 weight_to_token_ratio=0.093764 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244066760e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[4, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938328064 dispatch_meta_bytes=845104 combine_payload_bytes=938328064 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939173168 ingress_bytes=1466810368 link_bytes=1466810368 token_bytes=1877501232 weight_movement_bytes=528482304 total_nvlink_bytes=2405983536 weight_to_token_ratio=0.281482 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.213525784e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130798 payload_rows_remote=114414 dup_rows=131346 dup_rows_remote=114962 dedup_ratio=0.501045 router_tokens_per_expert=[8191, 0, 8190, 8190, 1, 0, 2, 1, 0, 8190, 41, 1, 0, 4, 1, 8161, 8182, 1, 1, 1, 1, 1, 1, 8187, 1, 1, 1, 0, 1, 1, 8188, 0, 7929, 1, 489, 2, 1, 0, 1, 0, 0, 0, 1, 8186, 0, 1, 7, 2743, 0, 3, 8187, 6777, 0, 25, 0, 6601, 0, 1, 8191, 3, 8191, 0, 0, 8191] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16112, 16384] dup_rows_per_dst=[16384, 16412, 16368, 16384, 16844, 21866, 10700, 16388] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16112, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127510 payload_rows_remote=111126 dup_rows=134634 dup_rows_remote=117382 dedup_ratio=0.513588 router_tokens_per_expert=[0, 0, 0, 8191, 1, 0, 11, 10, 0, 0, 3587, 8188, 0, 1, 1, 0, 212, 1, 89, 1, 215, 2, 0, 1, 0, 8156, 1, 1, 1, 38, 8190, 0, 1, 8190, 0, 0, 8190, 1, 0, 1, 116, 7258, 8059, 7233, 0, 8191, 8032, 1, 7922, 1, 537, 11, 1, 2, 0, 8180, 1, 1, 8183, 38, 0, 5833, 0, 8191] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16376, 16384, 12830] dup_rows_per_dst=[17252, 23552, 17104, 16390, 16382, 24704, 16924, 2326] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16376, 16384, 12830] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=938622976 dispatch_meta_bytes=914896 combine_payload_bytes=938622976 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=939537872 ingress_bytes=1290944512 link_bytes=1290944512 token_bytes=1878160848 weight_movement_bytes=352321536 total_nvlink_bytes=2230482384 weight_to_token_ratio=0.187589 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2667 gemm_tail_pad_fraction=0.017992 gemm_active_groups=21 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.228750400e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=924616 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940350408 ingress_bytes=1820229632 link_bytes=1820229632 token_bytes=1879776200 weight_movement_bytes=880803840 total_nvlink_bytes=2760580040 weight_to_token_ratio=0.468568 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.023674 gemm_active_groups=15 gemm_peak_group_rows=30464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019252614e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=917504 combine_payload_bytes=939212800 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940130304 ingress_bytes=1115373568 link_bytes=1115373568 token_bytes=1879343104 weight_movement_bytes=176160768 total_nvlink_bytes=2055503872 weight_to_token_ratio=0.093735 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245626401e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937279488 dispatch_meta_bytes=917504 combine_payload_bytes=937279488 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938196992 ingress_bytes=1113440256 link_bytes=1113440256 token_bytes=1875476480 weight_movement_bytes=176160768 total_nvlink_bytes=2051637248 weight_to_token_ratio=0.093929 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.234800661e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=910344192 dispatch_meta_bytes=914032 combine_payload_bytes=910344192 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=911258224 ingress_bytes=1262665728 link_bytes=1262665728 token_bytes=1821602416 weight_movement_bytes=352321536 total_nvlink_bytes=2173923952 weight_to_token_ratio=0.193413 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.070401014e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=103224 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 8191, 8192, 0, 0, 0, 0, 0, 8192, 53, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8080, 0, 11, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 2511, 0, 0, 8192, 5853, 0, 7, 0, 8064, 0, 0, 8192, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16382, 16488, 16384, 16384, 16184, 5018, 27848, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130834 payload_rows_remote=114450 dup_rows=131310 dup_rows_remote=114880 dedup_ratio=0.500908 router_tokens_per_expert=[8191, 0, 8185, 8191, 3, 0, 2, 1, 0, 8188, 47, 1, 0, 2, 0, 8169, 8169, 1, 1, 1, 1, 1, 1, 8190, 0, 1, 1, 0, 1, 1, 8188, 0, 7725, 1, 569, 3, 1, 0, 4, 0, 0, 0, 1, 8185, 0, 1, 3, 3503, 0, 2, 8188, 5962, 0, 43, 0, 6764, 0, 1, 8191, 6, 8191, 0, 0, 8191] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16148, 16384] dup_rows_per_dst=[16380, 16430, 16348, 16382, 16604, 23378, 9394, 16394] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16148, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=113120 dedup_ratio=0.500069 router_tokens_per_expert=[8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 22, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 8071, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 1701, 0, 0, 8192, 6574, 0, 57, 0, 8153, 0, 0, 8192, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16366, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15850, 15936, 16528, 16374, 16150, 3400, 28882, 17970] combine_rows_per_src=[16384, 16366, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114794 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 8189, 8192, 0, 0, 0, 0, 0, 8191, 64, 0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 8192, 0, 8109, 0, 26, 0, 0, 0, 4, 0, 0, 0, 0, 8190, 0, 0, 0, 1226, 0, 0, 8192, 6984, 0, 25, 0, 8154, 0, 0, 8192, 2, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16378, 16486, 16382, 16386, 16278, 15772, 17002, 16388] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=117818 dedup_ratio=0.500023 router_tokens_per_expert=[8190, 0, 8188, 8188, 2, 1, 6, 2, 2, 8189, 36, 2, 0, 0, 3, 8160, 8184, 0, 2, 2, 2, 0, 2, 8186, 1, 2, 2, 0, 2, 1, 8183, 0, 7583, 1, 843, 6, 2, 0, 10, 0, 0, 0, 2, 8187, 0, 2, 14, 3436, 0, 5, 8178, 5562, 1, 42, 1, 7086, 0, 1, 8189, 2, 8190, 1, 0, 8190] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16390, 16400, 16376, 16378, 16886, 13260, 19004, 16384] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114702 dedup_ratio=0.500046 router_tokens_per_expert=[8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 31, 0, 0, 0, 0, 8166, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 7827, 0, 66, 0, 0, 0, 4, 0, 0, 0, 0, 8134, 0, 0, 0, 2600, 0, 0, 8192, 6259, 0, 41, 0, 7833, 0, 0, 8192, 0, 8192, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16372, 16384] dup_rows_per_dst=[16384, 16394, 16382, 16384, 15794, 21468, 11894, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16372, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=871648 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940395744 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879919840 weight_movement_bytes=440401920 total_nvlink_bytes=2320321760 weight_to_token_ratio=0.234266 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=31232 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=937574400 dispatch_meta_bytes=917320 combine_payload_bytes=937574400 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=938491720 ingress_bytes=1377976320 link_bytes=1377976320 token_bytes=1876066120 weight_movement_bytes=440401920 total_nvlink_bytes=2316468040 weight_to_token_ratio=0.234748 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.716092196e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=911160 combine_payload_bytes=939376640 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940287800 ingress_bytes=1467858944 link_bytes=1467858944 token_bytes=1879664440 weight_movement_bytes=528482304 total_nvlink_bytes=2408146744 weight_to_token_ratio=0.281158 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219397372e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114660 dedup_ratio=0.500000 router_tokens_per_expert=[8161, 0, 21, 1, 1, 0, 8191, 8189, 7938, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 9, 1, 19, 1, 8189, 1, 1, 8185, 8190, 0, 19, 8191, 2, 8191, 8191, 8173, 0, 0, 0, 1, 1, 1, 8186, 8190, 5, 0, 4, 0, 34, 8190, 8191, 0, 1, 1, 221, 0, 0, 0, 1, 1, 8174, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16362, 15882, 16436, 16412, 16348, 16456, 16824, 16352] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917928 combine_payload_bytes=939524096 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940442024 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879966120 weight_movement_bytes=264241152 total_nvlink_bytes=2144207272 weight_to_token_ratio=0.140556 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130900 payload_rows_remote=114516 dup_rows=131244 dup_rows_remote=114320 dedup_ratio=0.500656 router_tokens_per_expert=[8191, 1, 8173, 8191, 2, 1, 7, 1, 0, 8187, 95, 1, 0, 0, 0, 8175, 8154, 1, 1, 1, 1, 0, 1, 8189, 0, 1, 1, 0, 1, 11, 8190, 0, 7139, 1, 471, 0, 1, 0, 17, 0, 0, 0, 1, 8190, 0, 1, 8, 5257, 1, 1, 8190, 4210, 1, 65, 0, 7348, 0, 1, 8190, 18, 8191, 2, 0, 8191] payload_rows_per_dst=[16384, 16384, 16380, 16382, 16254, 16384, 16350, 16382] dup_rows_per_dst=[16924, 16544, 16316, 16402, 16462, 26910, 6902, 14784] combine_rows_per_src=[16384, 16384, 16380, 16382, 16254, 16384, 16350, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=930000 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940404944 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879879888 weight_movement_bytes=176160768 total_nvlink_bytes=2056040656 weight_to_token_ratio=0.093709 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917512 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940343304 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879769096 weight_movement_bytes=176160768 total_nvlink_bytes=2055929864 weight_to_token_ratio=0.093714 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917392 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441488 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965584 weight_movement_bytes=528482304 total_nvlink_bytes=2408447888 weight_to_token_ratio=0.281113 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938115072 dispatch_meta_bytes=915344 combine_payload_bytes=938115072 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=939030416 ingress_bytes=1378516992 link_bytes=1378516992 token_bytes=1877145488 weight_movement_bytes=440401920 total_nvlink_bytes=2317547408 weight_to_token_ratio=0.234613 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.719119734e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=115146 dedup_ratio=0.500015 router_tokens_per_expert=[8156, 0, 15, 1, 1, 0, 8189, 8181, 7963, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 8, 1, 20, 1, 8189, 2, 1, 8186, 8190, 0, 48, 8190, 5, 8189, 8191, 8151, 0, 0, 0, 1, 1, 1, 8191, 8184, 3, 0, 7, 0, 115, 8190, 8191, 0, 1, 1, 125, 0, 0, 0, 1, 1, 8176, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16324, 15930, 16432, 16480, 16304, 16618, 16632, 16356] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114366 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 8190, 7543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 1, 0, 491, 8192, 8192, 0, 0, 0, 161, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 15086, 16384, 16384, 16384, 17364, 16706, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114528 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 8190, 7899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8187, 0, 0, 1, 0, 153, 8192, 8192, 0, 0, 0, 164, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 15798, 16346, 16384, 16226, 15994, 17400, 16544] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114694 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8191, 8190, 7603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 8192, 8189, 0, 0, 0, 0, 0, 0, 8191, 8189, 0, 0, 1, 0, 354, 8192, 8192, 0, 0, 0, 243, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16380, 15206, 16382, 16386, 16378, 17086, 16870, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 8192, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 8192, 0, 8192, 8192, 8191, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 140, 8192, 8192, 0, 0, 0, 152, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 15802, 16384, 16384, 16382, 16664, 16688, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114626 dedup_ratio=0.500015 router_tokens_per_expert=[8039, 0, 16, 0, 2, 0, 8189, 8188, 7873, 2, 2, 0, 0, 1, 2, 2, 0, 1, 0, 33, 2, 26, 2, 8188, 1, 2, 8176, 8189, 0, 142, 8190, 12, 8188, 8190, 8154, 0, 0, 0, 0, 2, 2, 8180, 8177, 12, 0, 3, 0, 43, 8189, 8190, 0, 48, 2, 249, 0, 1, 0, 2, 2, 8154, 1, 2, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[16106, 15760, 16498, 16660, 16308, 16450, 16974, 16320] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=919320 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940410648 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879901976 weight_movement_bytes=352321536 total_nvlink_bytes=2232223512 weight_to_token_ratio=0.187415 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 4] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916216 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940440312 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879964408 weight_movement_bytes=440401920 total_nvlink_bytes=2320366328 weight_to_token_ratio=0.234261 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916864 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940440960 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965056 weight_movement_bytes=352321536 total_nvlink_bytes=2232286592 weight_to_token_ratio=0.187409 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131012 payload_rows_remote=114628 dup_rows=131132 dup_rows_remote=112840 dedup_ratio=0.500229 router_tokens_per_expert=[0, 8187, 0, 46, 1, 1917, 1, 1, 0, 1, 0, 0, 0, 0, 8190, 55, 8167, 8165, 8189, 0, 0, 0, 5932, 0, 1, 8191, 954, 0, 1, 1, 8190, 8191, 0, 1, 0, 1, 1, 0, 0, 1, 1, 8191, 8187, 1, 8191, 8157, 1, 5567, 41, 8105, 0, 1, 1, 1, 4, 0, 6, 1, 2037, 2, 8184, 0, 7, 1] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16384, 16328, 16384, 16384] dup_rows_per_dst=[20252, 16492, 11816, 18292, 16388, 11126, 16300, 20466] combine_rows_per_src=[16384, 16382, 16382, 16384, 16384, 16328, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917528 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940441624 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965720 weight_movement_bytes=528482304 total_nvlink_bytes=2408448024 weight_to_token_ratio=0.281113 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.023674 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130894 payload_rows_remote=114510 dup_rows=131250 dup_rows_remote=114494 dedup_ratio=0.500679 router_tokens_per_expert=[8165, 0, 1, 1, 1, 0, 8187, 8170, 7800, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 17, 1, 6, 0, 8161, 12, 1, 8187, 8191, 0, 34, 8191, 6, 8191, 8191, 8180, 0, 0, 0, 1, 1, 1, 8190, 8170, 3, 0, 22, 0, 291, 8189, 8191, 0, 0, 1, 138, 0, 0, 0, 1, 1, 8171, 1, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16338, 16384, 16384, 16384, 16252, 16384] dup_rows_per_dst=[16756, 21608, 16256, 15982, 16750, 14440, 13110, 16348] combine_rows_per_src=[16384, 16384, 16338, 16384, 16384, 16384, 16252, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917240 combine_payload_bytes=939491328 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940408568 ingress_bytes=1379893248 link_bytes=1379893248 token_bytes=1879899896 weight_movement_bytes=440401920 total_nvlink_bytes=2320301816 weight_to_token_ratio=0.234269 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726826193e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939032576 dispatch_meta_bytes=909872 combine_payload_bytes=939032576 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=939942448 ingress_bytes=1467514880 link_bytes=1467514880 token_bytes=1878975024 weight_movement_bytes=528482304 total_nvlink_bytes=2407457328 weight_to_token_ratio=0.281261 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=30848 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.217470757e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938065920 dispatch_meta_bytes=916016 combine_payload_bytes=938065920 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938981936 ingress_bytes=1290387456 link_bytes=1290387456 token_bytes=1877047856 weight_movement_bytes=352321536 total_nvlink_bytes=2229369392 weight_to_token_ratio=0.187700 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.225631119e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114686 dup_rows=131076 dup_rows_remote=114650 dedup_ratio=0.500015 router_tokens_per_expert=[0, 8188, 0, 43, 1, 1338, 1, 1, 0, 1, 0, 0, 0, 0, 8187, 25, 8069, 8154, 8189, 0, 0, 0, 7014, 0, 1, 8191, 1411, 0, 1, 1, 8190, 8191, 0, 1, 0, 1, 1, 0, 0, 0, 1, 8191, 8189, 2, 8191, 8169, 1, 5394, 44, 8084, 0, 1, 1, 1, 2, 0, 12, 1, 1399, 1, 8182, 0, 5, 1] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[19068, 16426, 13784, 19206, 16386, 27128, 16260, 2818] combine_rows_per_src=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114694 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 4589, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 8192, 8192, 0, 0, 0, 4724, 0, 0, 8192, 907, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 2838, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 3328, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[9178, 16384, 25828, 18198, 16384, 5676, 16378, 23046] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=107064 dedup_ratio=0.500000 router_tokens_per_expert=[0, 8192, 0, 0, 0, 2546, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 8171, 8192, 8192, 0, 0, 0, 1874, 0, 0, 8192, 1486, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 6687, 0, 8186, 0, 0, 0, 0, 0, 0, 2, 0, 3810, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[5092, 15950, 19468, 8432, 16372, 24756, 16994, 24008] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=114712 dedup_ratio=0.500092 router_tokens_per_expert=[0, 8192, 0, 0, 0, 2965, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8167, 8192, 8192, 0, 0, 0, 5960, 0, 0, 8192, 1654, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 1, 8192, 8192, 0, 4544, 0, 8183, 0, 0, 0, 0, 0, 0, 8, 0, 1287, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16360, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[22314, 16382, 11894, 19692, 16384, 9090, 16366, 18974] combine_rows_per_src=[16384, 16384, 16360, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130948 payload_rows_remote=114666 dup_rows=131196 dup_rows_remote=119288 dedup_ratio=0.500473 router_tokens_per_expert=[0, 8186, 1, 73, 2, 2404, 1, 0, 0, 9, 0, 0, 0, 0, 8189, 107, 8161, 8141, 8185, 0, 0, 0, 5547, 1, 2, 8190, 1037, 0, 2, 5, 8189, 8190, 0, 2, 0, 3, 2, 0, 0, 0, 2, 8190, 8179, 1, 8189, 8135, 2, 5955, 44, 8038, 0, 4, 0, 2, 3, 0, 6, 2, 1496, 3, 8173, 0, 17, 2] payload_rows_per_dst=[16384, 16380, 16366, 16384, 16384, 16282, 16384, 16384] dup_rows_per_dst=[14024, 16610, 18260, 18466, 16388, 11908, 16168, 19372] combine_rows_per_src=[16384, 16380, 16366, 16384, 16384, 16282, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114688 dup_rows=131080 dup_rows_remote=120760 dedup_ratio=0.500031 router_tokens_per_expert=[0, 8192, 0, 0, 0, 2479, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 8185, 8192, 8192, 0, 0, 0, 5163, 0, 0, 8192, 1642, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 5227, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[21342, 16390, 10320, 19668, 16384, 11202, 16382, 19392] combine_rows_per_src=[16384, 16384, 16376, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917344 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940425056 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879932768 weight_movement_bytes=176160768 total_nvlink_bytes=2056093536 weight_to_token_ratio=0.093706 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131016 payload_rows_remote=114632 dup_rows=131128 dup_rows_remote=105364 dedup_ratio=0.500214 router_tokens_per_expert=[1, 4812, 1, 100, 1, 0, 1, 0, 8092, 0, 1, 0, 0, 6448, 1, 5, 1, 0, 1, 1000, 1, 1, 4, 1, 4676, 1, 10, 8187, 8, 0, 1, 8191, 7590, 0, 0, 8184, 0, 1, 0, 36, 4, 8191, 8183, 8148, 1, 8135, 8188, 0, 2, 8183, 8190, 1, 1, 98, 1, 0, 1, 8191, 8191, 0, 0, 5, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16328, 16384, 16384] dup_rows_per_dst=[20268, 12710, 18382, 25764, 15238, 5804, 16568, 16394] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16328, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131056 payload_rows_remote=114672 dup_rows=131088 dup_rows_remote=111286 dedup_ratio=0.500061 router_tokens_per_expert=[0, 8187, 0, 33, 1, 1689, 1, 1, 0, 9, 0, 0, 0, 0, 8179, 36, 8131, 8181, 8191, 0, 0, 0, 5156, 0, 1, 8191, 1149, 0, 1, 2, 8190, 8191, 0, 1, 0, 1, 1, 0, 0, 0, 1, 8191, 8184, 12, 8191, 8167, 1, 6999, 56, 8053, 0, 1, 1, 1, 1, 0, 24, 0, 1470, 1, 8190, 0, 4, 1] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16384, 16372, 16384, 16384] dup_rows_per_dst=[19802, 16448, 19758, 9118, 16386, 30356, 16224, 2996] combine_rows_per_src=[16384, 16382, 16382, 16384, 16384, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=917504 combine_payload_bytes=939327488 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940244992 ingress_bytes=1291649024 link_bytes=1291649024 token_bytes=1879572480 weight_movement_bytes=352321536 total_nvlink_bytes=2231894016 weight_to_token_ratio=0.187448 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232695374e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917528 combine_payload_bytes=939524096 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940441624 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965720 weight_movement_bytes=352321536 total_nvlink_bytes=2232287256 weight_to_token_ratio=0.187408 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=887008 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940411104 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879935200 weight_movement_bytes=176160768 total_nvlink_bytes=2056095968 weight_to_token_ratio=0.093706 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939343872 dispatch_meta_bytes=935816 combine_payload_bytes=939343872 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940279688 ingress_bytes=1643986944 link_bytes=1643986944 token_bytes=1879623560 weight_movement_bytes=704643072 total_nvlink_bytes=2584266632 weight_to_token_ratio=0.374885 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205640652e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=941792 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940465888 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1879989984 weight_movement_bytes=704643072 total_nvlink_bytes=2584633056 weight_to_token_ratio=0.374812 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939065344 dispatch_meta_bytes=879984 combine_payload_bytes=939065344 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939945328 ingress_bytes=1643708416 link_bytes=1643708416 token_bytes=1879010672 weight_movement_bytes=704643072 total_nvlink_bytes=2583653744 weight_to_token_ratio=0.375007 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=26880 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204081012e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939393024 dispatch_meta_bytes=903832 combine_payload_bytes=939393024 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940296856 ingress_bytes=1115553792 link_bytes=1115553792 token_bytes=1879689880 weight_movement_bytes=176160768 total_nvlink_bytes=2055850648 weight_to_token_ratio=0.093718 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246635580e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131004 payload_rows_remote=114620 dup_rows=131140 dup_rows_remote=113760 dedup_ratio=0.500259 router_tokens_per_expert=[1, 3464, 1, 102, 1, 0, 1, 0, 8093, 0, 1, 0, 0, 7499, 0, 13, 3, 0, 1, 867, 1, 1, 20, 1, 4852, 1, 17, 8187, 5, 0, 1, 8191, 7664, 0, 0, 8186, 0, 1, 2, 30, 3, 8191, 8179, 8157, 1, 8098, 8188, 0, 5, 8179, 8190, 6, 1, 283, 1, 0, 1, 8190, 8191, 0, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16316, 16384, 16384] dup_rows_per_dst=[23328, 17380, 18144, 23572, 15382, 6, 16946, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16316, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114658 dup_rows=131102 dup_rows_remote=115098 dedup_ratio=0.500114 router_tokens_per_expert=[0, 4983, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 6443, 0, 7, 0, 0, 0, 477, 0, 0, 1, 0, 4653, 0, 2, 8192, 0, 0, 0, 8192, 8002, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8191, 8143, 0, 8192, 8191, 0, 1, 8192, 8192, 2, 0, 58, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16354, 16384, 16384] dup_rows_per_dst=[14320, 12896, 17338, 20514, 16004, 17140, 16506, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16354, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127822 payload_rows_remote=111438 dup_rows=134322 dup_rows_remote=117910 dedup_ratio=0.512398 router_tokens_per_expert=[0, 4793, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 7283, 0, 7, 0, 0, 0, 612, 0, 0, 4, 0, 4661, 0, 0, 8192, 0, 0, 0, 8192, 7254, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 8148, 0, 8191, 8192, 0, 0, 8191, 8192, 0, 0, 26, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 13134, 16384, 16384, 16384, 16384] dup_rows_per_dst=[9586, 14544, 6136, 4202, 22878, 44158, 16406, 16412] combine_rows_per_src=[16384, 16384, 16384, 13134, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=99330 dedup_ratio=0.500023 router_tokens_per_expert=[2, 4767, 2, 177, 0, 1, 3, 0, 7970, 0, 2, 0, 0, 7462, 0, 5, 8, 0, 2, 1318, 2, 2, 2, 4, 3521, 3, 6, 8181, 12, 0, 2, 8190, 7437, 0, 0, 8177, 0, 0, 1, 73, 7, 8190, 8178, 8088, 2, 8078, 8181, 0, 1, 8184, 8188, 2, 1, 254, 2, 0, 2, 8182, 8189, 1, 0, 5, 0, 5] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16382, 16384] dup_rows_per_dst=[9882, 14494, 19028, 7664, 14996, 31748, 16882, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114680 dedup_ratio=0.500000 router_tokens_per_expert=[0, 4295, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 6955, 0, 1, 0, 0, 0, 336, 0, 0, 3, 0, 5034, 0, 0, 8192, 0, 0, 0, 8192, 8036, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 8107, 0, 8191, 8192, 0, 1, 8192, 8192, 0, 0, 3, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[8590, 13908, 16926, 10068, 16072, 32732, 16392, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130736 payload_rows_remote=114352 dup_rows=131408 dup_rows_remote=114628 dedup_ratio=0.501282 router_tokens_per_expert=[0, 4523, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 6810, 0, 0, 0, 0, 0, 196, 0, 0, 2, 0, 5534, 0, 0, 8192, 0, 0, 0, 8192, 7707, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 8024, 0, 8155, 8192, 0, 0, 8192, 8192, 0, 0, 24, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16048, 16384, 16384] dup_rows_per_dst=[25356, 13590, 16780, 27452, 15414, 0, 16432, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16048, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=938967040 dispatch_meta_bytes=913520 combine_payload_bytes=938967040 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939880560 ingress_bytes=1115127808 link_bytes=1115127808 token_bytes=1878847600 weight_movement_bytes=176160768 total_nvlink_bytes=2055008368 weight_to_token_ratio=0.093760 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244250248e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=919024 combine_payload_bytes=939278336 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940197360 ingress_bytes=1115439104 link_bytes=1115439104 token_bytes=1879475696 weight_movement_bytes=176160768 total_nvlink_bytes=2055636464 weight_to_token_ratio=0.093729 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.025568 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245993375e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=912900096 dispatch_meta_bytes=917392 combine_payload_bytes=912900096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=913817488 ingress_bytes=1089060864 link_bytes=1089060864 token_bytes=1826717584 weight_movement_bytes=176160768 total_nvlink_bytes=2002878352 weight_to_token_ratio=0.096436 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.098286242e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130888 payload_rows_remote=114504 dup_rows=131256 dup_rows_remote=114830 dedup_ratio=0.500702 router_tokens_per_expert=[8188, 0, 0, 0, 60, 8191, 8173, 0, 1, 1, 0, 8189, 2, 0, 8132, 1, 8191, 0, 14, 0, 0, 1353, 85, 8137, 0, 1, 0, 8, 12, 1, 8191, 1, 11, 0, 1, 2, 2, 0, 1, 1, 0, 2, 1, 1, 0, 0, 8182, 8191, 0, 8190, 8190, 0, 1, 8179, 1, 0, 1, 8191, 0, 7754, 7237, 0, 1, 0] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 16384, 16384, 16204] dup_rows_per_dst=[16464, 16268, 19178, 16426, 16410, 16370, 16358, 13782] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 16384, 16384, 16204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131036 payload_rows_remote=114652 dup_rows=131108 dup_rows_remote=111366 dedup_ratio=0.500137 router_tokens_per_expert=[1, 1656, 1, 86, 1, 0, 1, 0, 8014, 0, 1, 0, 0, 7850, 0, 47, 5, 0, 1, 989, 1, 1, 9, 1, 6366, 1, 56, 8187, 11, 0, 1, 8191, 7524, 0, 0, 8187, 0, 0, 0, 27, 4, 8191, 8163, 8174, 1, 8129, 8188, 0, 21, 8171, 8191, 16, 2, 238, 1, 0, 1, 8172, 8191, 0, 0, 2, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16350, 16384, 16384] dup_rows_per_dst=[19742, 31550, 18338, 15086, 15094, 8, 14942, 16348] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16350, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=856048 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940330992 ingress_bytes=1820278784 link_bytes=1820278784 token_bytes=1879805936 weight_movement_bytes=880803840 total_nvlink_bytes=2760609776 weight_to_token_ratio=0.468561 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019280137e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917472 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=940441568 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965664 weight_movement_bytes=352321536 total_nvlink_bytes=2232287200 weight_to_token_ratio=0.187408 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=936771584 dispatch_meta_bytes=915920 combine_payload_bytes=936771584 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937687504 ingress_bytes=1289093120 link_bytes=1289093120 token_bytes=1874459088 weight_movement_bytes=352321536 total_nvlink_bytes=2226780624 weight_to_token_ratio=0.187959 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.218383378e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938016768 dispatch_meta_bytes=917336 combine_payload_bytes=938016768 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938934104 ingress_bytes=1114177536 link_bytes=1114177536 token_bytes=1876950872 weight_movement_bytes=176160768 total_nvlink_bytes=2053111640 weight_to_token_ratio=0.093855 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.238929121e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=904072 combine_payload_bytes=939229184 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940133256 ingress_bytes=1291550720 link_bytes=1291550720 token_bytes=1879362440 weight_movement_bytes=352321536 total_nvlink_bytes=2231683976 weight_to_token_ratio=0.187469 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232144912e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130896 payload_rows_remote=114512 dup_rows=131248 dup_rows_remote=114990 dedup_ratio=0.500671 router_tokens_per_expert=[8184, 0, 0, 0, 56, 8191, 8173, 0, 1, 1, 0, 8185, 2, 0, 8131, 1, 8191, 0, 10, 0, 0, 940, 80, 8180, 0, 1, 0, 11, 38, 1, 8190, 1, 15, 0, 1, 1, 2, 6, 1, 1, 0, 5, 1, 1, 0, 0, 8180, 8191, 0, 8190, 8190, 0, 1, 8147, 1, 0, 1, 8191, 0, 7749, 7627, 1, 1, 0] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 16384, 16384, 16212] dup_rows_per_dst=[16456, 16258, 18420, 16482, 16420, 16372, 16294, 14546] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 16384, 16384, 16212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114694 dedup_ratio=0.500038 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8191, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 364, 0, 8182, 0, 0, 0, 0, 119, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 0, 8143, 0, 0, 0, 8192, 0, 8117, 7841, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] dup_rows_per_dst=[16384, 16382, 17092, 16622, 16388, 16386, 16286, 15542] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130916 payload_rows_remote=114532 dup_rows=131228 dup_rows_remote=104896 dedup_ratio=0.500595 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8189, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 531, 0, 8181, 0, 0, 0, 0, 42, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 0, 8152, 0, 0, 0, 8192, 0, 8055, 7808, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16228, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15920, 16378, 7168, 15730, 16388, 16384, 16928, 26332] combine_rows_per_src=[16384, 16384, 16228, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130842 payload_rows_remote=114458 dup_rows=131302 dup_rows_remote=114984 dedup_ratio=0.500877 router_tokens_per_expert=[8186, 0, 0, 0, 96, 8189, 8158, 0, 2, 4, 0, 8186, 4, 0, 8084, 2, 8190, 0, 48, 0, 0, 768, 120, 8181, 0, 2, 1, 22, 15, 2, 8189, 2, 2, 0, 2, 4, 6, 1, 2, 2, 0, 1, 0, 2, 2, 0, 8156, 8190, 2, 8188, 8188, 0, 2, 8183, 1, 0, 2, 8189, 0, 7623, 7871, 1, 1, 0] payload_rows_per_dst=[16380, 16384, 16380, 16384, 16384, 16384, 16384, 16162] dup_rows_per_dst=[16504, 16180, 18234, 16460, 16404, 16318, 16368, 14834] combine_rows_per_src=[16380, 16384, 16380, 16384, 16384, 16384, 16384, 16162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=114724 dedup_ratio=0.500008 router_tokens_per_expert=[8190, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 283, 0, 8170, 0, 0, 0, 0, 11, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 0, 8175, 0, 0, 0, 8192, 0, 8173, 7958, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16384, 16384, 16906, 16406, 16380, 16384, 16350, 15880] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113668 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 512, 0, 8190, 0, 0, 0, 0, 14, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 8192, 0, 0, 8147, 0, 0, 0, 8192, 0, 8173, 7733, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16384, 16382, 17404, 16412, 16384, 16384, 16294, 15428] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938082304 dispatch_meta_bytes=918008 combine_payload_bytes=938082304 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939000312 ingress_bytes=1114243072 link_bytes=1114243072 token_bytes=1877082616 weight_movement_bytes=176160768 total_nvlink_bytes=2053243384 weight_to_token_ratio=0.093848 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239296095e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130830 payload_rows_remote=114446 dup_rows=131314 dup_rows_remote=114926 dedup_ratio=0.500923 router_tokens_per_expert=[7355, 8181, 0, 1, 0, 1, 7827, 0, 1, 1, 1, 1, 1, 0, 8189, 1, 8101, 0, 1, 0, 0, 8191, 182, 8119, 0, 0, 8191, 1, 2, 0, 0, 1, 16, 8190, 0, 8191, 7287, 1, 7, 8190, 192, 1729, 0, 3, 0, 2, 8161, 14, 1, 8191, 0, 1, 1, 0, 8190, 0, 1, 0, 0, 0, 0, 163, 8191, 1] payload_rows_per_dst=[16144, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[14224, 16388, 16422, 16388, 14618, 20200, 16384, 16690] combine_rows_per_src=[16144, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917488 combine_payload_bytes=939442176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940359664 ingress_bytes=1291763712 link_bytes=1291763712 token_bytes=1879801840 weight_movement_bytes=352321536 total_nvlink_bytes=2232123376 weight_to_token_ratio=0.187425 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233337578e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130854 payload_rows_remote=114472 dup_rows=131290 dup_rows_remote=114704 dedup_ratio=0.500832 router_tokens_per_expert=[8190, 0, 0, 0, 54, 8191, 8173, 0, 1, 2, 0, 8172, 2, 0, 8131, 1, 8191, 0, 3, 1, 0, 460, 74, 8179, 0, 1, 0, 9, 69, 1, 8191, 1, 10, 0, 1, 1, 2, 10, 1, 1, 0, 15, 0, 1, 0, 0, 8188, 8191, 1, 8191, 8190, 0, 1, 8135, 0, 0, 1, 8187, 0, 7860, 7983, 4, 1, 0] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 16378, 16314, 16246] dup_rows_per_dst=[16586, 16840, 16836, 16534, 16298, 16480, 16264, 15452] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 16378, 16314, 16246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938246144 dispatch_meta_bytes=877712 combine_payload_bytes=938246144 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=939123856 ingress_bytes=1290567680 link_bytes=1290567680 token_bytes=1877370000 weight_movement_bytes=352321536 total_nvlink_bytes=2229691536 weight_to_token_ratio=0.187668 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.226640299e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=937639936 dispatch_meta_bytes=917768 combine_payload_bytes=937639936 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=938557704 ingress_bytes=1113800704 link_bytes=1113800704 token_bytes=1876197640 weight_movement_bytes=176160768 total_nvlink_bytes=2052358408 weight_to_token_ratio=0.093892 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.236819019e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=917640 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940425352 ingress_bytes=1379909632 link_bytes=1379909632 token_bytes=1879933064 weight_movement_bytes=440401920 total_nvlink_bytes=2320334984 weight_to_token_ratio=0.234265 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726917937e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=913424 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=528482304 egress_bytes=940437520 ingress_bytes=1556086784 link_bytes=1556086784 token_bytes=1879961616 weight_movement_bytes=616562688 total_nvlink_bytes=2496524304 weight_to_token_ratio=0.327966 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713436448e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937541632 dispatch_meta_bytes=917488 combine_payload_bytes=937541632 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938459120 ingress_bytes=1113702400 link_bytes=1113702400 token_bytes=1876000752 weight_movement_bytes=176160768 total_nvlink_bytes=2052161520 weight_to_token_ratio=0.093902 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.236268558e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=937754624 dispatch_meta_bytes=916704 combine_payload_bytes=937754624 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938671328 ingress_bytes=1466236928 link_bytes=1466236928 token_bytes=1876425952 weight_movement_bytes=528482304 total_nvlink_bytes=2404908256 weight_to_token_ratio=0.281643 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.210314759e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130834 payload_rows_remote=114450 dup_rows=131310 dup_rows_remote=114926 dedup_ratio=0.500908 router_tokens_per_expert=[7579, 8179, 2, 0, 0, 1, 7741, 0, 1, 1, 1, 1, 1, 0, 8187, 1, 8104, 0, 1, 0, 0, 8189, 173, 8079, 0, 0, 8191, 1, 1, 0, 0, 2, 14, 8191, 0, 8191, 7067, 1, 4, 8190, 205, 1603, 0, 20, 0, 2, 8177, 38, 1, 8191, 0, 1, 1, 0, 8190, 0, 1, 1, 0, 0, 0, 355, 8191, 1] payload_rows_per_dst=[16148, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14498, 16384, 16332, 16384, 22034, 12222, 16384, 17072] combine_rows_per_src=[16148, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=117486 dedup_ratio=0.500008 router_tokens_per_expert=[8132, 8190, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8191, 2, 8188, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 8192, 0, 8192, 6794, 0, 0, 8192, 50, 1175, 1, 6, 0, 0, 8192, 107, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 141, 8192, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16230, 16384, 16380, 16384, 13588, 19062, 16384, 16662] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=113902 dedup_ratio=0.500000 router_tokens_per_expert=[8121, 8192, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 8179, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 6690, 0, 0, 8192, 10, 1125, 0, 11, 0, 0, 8191, 114, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 393, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16110, 16376, 16358, 16384, 13386, 18902, 16386, 17170] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130936 payload_rows_remote=114552 dup_rows=131208 dup_rows_remote=110282 dedup_ratio=0.500519 router_tokens_per_expert=[7354, 8178, 7, 1, 0, 0, 7703, 0, 3, 2, 2, 2, 2, 0, 8183, 2, 8030, 0, 2, 1, 0, 8189, 243, 7943, 0, 1, 8190, 3, 2, 0, 0, 1, 54, 8188, 1, 8190, 7214, 2, 10, 8188, 250, 2089, 0, 12, 0, 3, 8081, 31, 2, 8190, 0, 2, 2, 0, 8187, 0, 2, 2, 2, 0, 0, 135, 8189, 2] payload_rows_per_dst=[16248, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[13882, 16388, 16054, 16388, 14552, 20926, 16382, 16636] combine_rows_per_src=[16248, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=10 export_slots=10 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 3, 2, 0, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8164, 8191, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 0, 8185, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 7005, 0, 0, 8192, 25, 1022, 0, 2, 0, 0, 8192, 20, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 161, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16314, 16384, 16370, 16384, 14010, 18522, 16384, 16704] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114684 dedup_ratio=0.500000 router_tokens_per_expert=[8159, 8192, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 8192, 4, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 7336, 0, 0, 8192, 15, 669, 0, 0, 0, 0, 8192, 38, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 272, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16104, 16384, 16388, 16384, 14672, 17828, 16384, 16928] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=937574400 dispatch_meta_bytes=917504 combine_payload_bytes=937574400 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=938491904 ingress_bytes=1201815552 link_bytes=1201815552 token_bytes=1876066304 weight_movement_bytes=264241152 total_nvlink_bytes=2140307456 weight_to_token_ratio=0.140849 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.729665429e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=928688 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=880803840 egress_bytes=940436400 ingress_bytes=1820311552 link_bytes=1820311552 token_bytes=1879944112 weight_movement_bytes=880803840 total_nvlink_bytes=2760747952 weight_to_token_ratio=0.468527 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019298486e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114684 dup_rows=131078 dup_rows_remote=114476 dedup_ratio=0.500023 router_tokens_per_expert=[8143, 6838, 1, 1290, 1, 81, 8004, 0, 8190, 1, 78, 8042, 2, 0, 1, 0, 8191, 0, 60, 1, 1, 8190, 0, 3, 0, 0, 1, 114, 0, 8187, 0, 8190, 0, 1, 1, 8190, 0, 7, 1, 0, 1, 8109, 1, 0, 0, 221, 1, 1, 4, 1, 8166, 0, 0, 1, 8191, 0, 0, 8184, 0, 8190, 8188, 0, 2, 1] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16380] dup_rows_per_dst=[16046, 16244, 16508, 16602, 16396, 16570, 16342, 16370] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=106752 dedup_ratio=0.500015 router_tokens_per_expert=[7560, 8154, 4, 1, 0, 0, 8010, 0, 1, 1, 1, 1, 1, 0, 8181, 0, 8114, 0, 1, 0, 0, 8181, 206, 7960, 0, 3, 8191, 1, 2, 0, 0, 11, 15, 8187, 0, 8191, 6955, 1, 6, 8190, 214, 1372, 1, 34, 0, 2, 8175, 99, 1, 8191, 0, 1, 1, 0, 8190, 0, 1, 1, 1, 0, 0, 466, 8191, 1] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[24324, 16384, 16770, 16376, 24218, 3410, 16384, 13210] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=914360 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940438456 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879962552 weight_movement_bytes=176160768 total_nvlink_bytes=2056123320 weight_to_token_ratio=0.093704 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938409984 dispatch_meta_bytes=899336 combine_payload_bytes=938409984 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939309320 ingress_bytes=1114570752 link_bytes=1114570752 token_bytes=1877719304 weight_movement_bytes=176160768 total_nvlink_bytes=2053880072 weight_to_token_ratio=0.093816 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.241130966e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 2, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1027604480 link_bytes=1027604480 token_bytes=1879965696 weight_movement_bytes=88080384 total_nvlink_bytes=1968046080 weight_to_token_ratio=0.046852 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.023674 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.754156145e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917488 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940441584 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965680 weight_movement_bytes=352321536 total_nvlink_bytes=2232287216 weight_to_token_ratio=0.187408 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=916640 combine_payload_bytes=939491328 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=940407968 ingress_bytes=1644134400 link_bytes=1644134400 token_bytes=1879899296 weight_movement_bytes=704643072 total_nvlink_bytes=2584542368 weight_to_token_ratio=0.374830 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.024621 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206466345e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=885744 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940377072 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879868400 weight_movement_bytes=352321536 total_nvlink_bytes=2232189936 weight_to_token_ratio=0.187418 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114668 dup_rows=131092 dup_rows_remote=114888 dedup_ratio=0.500076 router_tokens_per_expert=[8176, 7264, 1, 679, 1, 55, 7924, 0, 8188, 1, 53, 8050, 2, 0, 0, 0, 8191, 0, 54, 5, 1, 8190, 0, 2, 0, 0, 2, 365, 0, 8188, 0, 8190, 0, 1, 1, 8182, 0, 5, 1, 0, 1, 8134, 1, 0, 0, 234, 1, 1, 4, 1, 8167, 0, 0, 1, 8191, 0, 0, 8184, 0, 8188, 8189, 0, 2, 1] payload_rows_per_dst=[16368, 16384, 16384, 16382, 16384, 16384, 16384, 16382] dup_rows_per_dst=[15480, 16204, 16502, 17108, 16372, 16712, 16344, 16370] combine_rows_per_src=[16368, 16384, 16384, 16382, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114692 dedup_ratio=0.500000 router_tokens_per_expert=[8158, 7369, 0, 790, 0, 0, 8191, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 70, 0, 8192, 0, 8192, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 8191, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16316, 16382, 16386, 16524, 16380, 16316, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29926636248 prefetch_bytes=2290089984 reduce_bytes=5108662272 weight_movement_bytes=7398752256 egress_bytes=14970649304 ingress_bytes=22354739200 link_bytes=22354739200 total_nvlink_bytes=37325388504 weight_to_token_ratio=0.247230 prefetch_share_of_ingress=0.102443 dedup_ratio=0.502476 prefetch_slots=52 unused_prefetch_slots=460 slot_utilization=0.101562 reduce_slots=58 remote_demand=26 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021544 gemm_inner_pad_upper_bound=32385 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.823575512e-05 prefetch_bytes_per_gemm_flop=8.014717487e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114548 dedup_ratio=0.500000 router_tokens_per_expert=[8170, 5628, 0, 2670, 0, 0, 8093, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 27, 0, 8192, 0, 8192, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16398, 16384, 16320, 16142, 16352, 16336, 16616, 16524] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114240 dedup_ratio=0.500023 router_tokens_per_expert=[8172, 7217, 2, 872, 1, 93, 7853, 0, 8189, 2, 123, 7908, 3, 0, 0, 0, 8190, 0, 114, 1, 2, 8189, 1, 5, 0, 2, 2, 218, 0, 8183, 0, 8189, 0, 2, 1, 8186, 1, 20, 0, 0, 2, 8067, 2, 0, 0, 364, 2, 2, 7, 3, 8139, 0, 0, 3, 8190, 0, 0, 8170, 1, 8188, 8185, 0, 4, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16378] dup_rows_per_dst=[15692, 16066, 16620, 16804, 16412, 16838, 16300, 16346] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939360256 dispatch_meta_bytes=918224 combine_payload_bytes=939360256 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940278480 ingress_bytes=1115521024 link_bytes=1115521024 token_bytes=1879638736 weight_movement_bytes=176160768 total_nvlink_bytes=2055799504 weight_to_token_ratio=0.093721 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.023674 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246452093e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8186, 6585, 0, 1714, 0, 0, 8075, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 21, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16364, 16384, 16384, 16426, 16384, 16362, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[8115, 6552, 0, 1730, 0, 0, 8174, 0, 8189, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16528, 16378, 16388, 16402, 16380, 16228, 16384, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940441616 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965712 weight_movement_bytes=264241152 total_nvlink_bytes=2144206864 weight_to_token_ratio=0.140556 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130878 payload_rows_remote=114494 dup_rows=131266 dup_rows_remote=113002 dedup_ratio=0.500740 router_tokens_per_expert=[8165, 7534, 1, 489, 1, 11, 7899, 0, 8184, 1, 63, 8061, 2, 0, 0, 0, 8191, 1, 12, 11, 1, 8190, 0, 2, 0, 0, 2, 391, 0, 8189, 0, 8190, 0, 1, 1, 8173, 3, 0, 0, 0, 1, 8181, 1, 0, 0, 202, 1, 1, 2, 1, 8155, 0, 0, 2, 8191, 0, 0, 8184, 0, 8188, 8189, 1, 2, 1] payload_rows_per_dst=[16384, 16384, 16384, 16206, 16380, 16384, 16384, 16372] dup_rows_per_dst=[18264, 17030, 15964, 15246, 16400, 15710, 16318, 16334] combine_rows_per_src=[16384, 16384, 16384, 16206, 16380, 16384, 16384, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=916944 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=940441040 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965136 weight_movement_bytes=440401920 total_nvlink_bytes=2320367056 weight_to_token_ratio=0.234261 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=915688 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940390632 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879865576 weight_movement_bytes=176160768 total_nvlink_bytes=2056026344 weight_to_token_ratio=0.093709 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29903246304 prefetch_bytes=3082813440 reduce_bytes=2642411520 weight_movement_bytes=5725224960 egress_bytes=14958908384 ingress_bytes=20669562880 link_bytes=20669562880 total_nvlink_bytes=35628471264 weight_to_token_ratio=0.191458 prefetch_share_of_ingress=0.149147 dedup_ratio=0.503010 prefetch_slots=70 unused_prefetch_slots=442 slot_utilization=0.136719 reduce_slots=30 remote_demand=35 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021484 gemm_inner_pad_upper_bound=31496 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.233807764e-05 prefetch_bytes_per_gemm_flop=1.078904277e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940441600 ingress_bytes=1115684864 link_bytes=1115684864 token_bytes=1879965696 weight_movement_bytes=176160768 total_nvlink_bytes=2056126464 weight_to_token_ratio=0.093704 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247369529e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940441600 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965696 weight_movement_bytes=352321536 total_nvlink_bytes=2232287232 weight_to_token_ratio=0.187408 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29942644712 prefetch_bytes=3435134976 reduce_bytes=2906652672 weight_movement_bytes=6341787648 egress_bytes=14978645992 ingress_bytes=21305786368 link_bytes=21305786368 total_nvlink_bytes=36284432360 weight_to_token_ratio=0.211798 prefetch_share_of_ingress=0.161230 dedup_ratio=0.502213 prefetch_slots=78 unused_prefetch_slots=434 slot_utilization=0.152344 reduce_slots=33 remote_demand=39 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021188 gemm_inner_pad_upper_bound=32766 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.456469387e-05 prefetch_bytes_per_gemm_flop=1.202207623e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29622626184 prefetch_bytes=2906652672 reduce_bytes=1761607680 weight_movement_bytes=4668260352 egress_bytes=14818518920 ingress_bytes=19472367616 link_bytes=19472367616 total_nvlink_bytes=34290886536 weight_to_token_ratio=0.157591 prefetch_share_of_ingress=0.149271 dedup_ratio=0.506826 prefetch_slots=66 unused_prefetch_slots=446 slot_utilization=0.128906 reduce_slots=20 remote_demand=33 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021307 gemm_inner_pad_upper_bound=33655 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.814820655e-05 prefetch_bytes_per_gemm_flop=1.017252604e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=937934848 dispatch_meta_bytes=909984 combine_payload_bytes=937934848 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=938844832 ingress_bytes=1290256384 link_bytes=1290256384 token_bytes=1876779680 weight_movement_bytes=352321536 total_nvlink_bytes=2229101216 weight_to_token_ratio=0.187727 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.021780 gemm_active_groups=18 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.224897171e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29940644848 prefetch_bytes=2554331136 reduce_bytes=3963617280 weight_movement_bytes=6517948416 egress_bytes=14977596400 ingress_bytes=21480996864 link_bytes=21480996864 total_nvlink_bytes=36458593264 weight_to_token_ratio=0.217696 prefetch_share_of_ingress=0.118911 dedup_ratio=0.502462 prefetch_slots=58 unused_prefetch_slots=454 slot_utilization=0.113281 reduce_slots=45 remote_demand=29 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021780 gemm_inner_pad_upper_bound=32004 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.517788489e-05 prefetch_bytes_per_gemm_flop=8.939492582e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29753598512 prefetch_bytes=2642411520 reduce_bytes=1937768448 weight_movement_bytes=4580179968 egress_bytes=14884069936 ingress_bytes=19449708544 link_bytes=19449708544 total_nvlink_bytes=34333778480 weight_to_token_ratio=0.153937 prefetch_share_of_ingress=0.135859 dedup_ratio=0.504872 prefetch_slots=60 unused_prefetch_slots=452 slot_utilization=0.117188 reduce_slots=22 remote_demand=30 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021366 gemm_inner_pad_upper_bound=33020 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.806890571e-05 prefetch_bytes_per_gemm_flop=9.247750947e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29924627960 prefetch_bytes=3082813440 reduce_bytes=3523215360 weight_movement_bytes=6606028800 egress_bytes=14969640440 ingress_bytes=21561016320 link_bytes=21561016320 total_nvlink_bytes=36530656760 weight_to_token_ratio=0.220756 prefetch_share_of_ingress=0.142981 dedup_ratio=0.502549 prefetch_slots=70 unused_prefetch_slots=442 slot_utilization=0.136719 reduce_slots=40 remote_demand=35 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021070 gemm_inner_pad_upper_bound=33020 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.545793211e-05 prefetch_bytes_per_gemm_flop=1.078904277e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29797152448 prefetch_bytes=3435134976 reduce_bytes=1585446912 weight_movement_bytes=5020581888 egress_bytes=14905767616 ingress_bytes=19911966720 link_bytes=19911966720 total_nvlink_bytes=34817734336 weight_to_token_ratio=0.168492 prefetch_share_of_ingress=0.172516 dedup_ratio=0.504217 prefetch_slots=78 unused_prefetch_slots=434 slot_utilization=0.152344 reduce_slots=18 remote_demand=39 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.021544 gemm_inner_pad_upper_bound=32512 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.968668873e-05 prefetch_bytes_per_gemm_flop=1.202207623e-05 anomalies=none - [2026-09-16 21:39:35] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 608.4 | throughput per GPU (TFLOP/s/GPU): 547.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.994638E+00 | loss scale: 1.0 | grad norm: 5.228 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120210 payload_rows_remote=105384 dup_rows=141934 dup_rows_remote=128240 dedup_ratio=0.541435 router_tokens_per_expert=[4726, 1827, 1700, 1379, 3282, 0, 1960, 2308, 1920, 5087, 3857, 2189, 2543, 3872, 11, 688, 708, 1951, 1751, 2180, 6, 1472, 4165, 2027, 778, 290, 4352, 1585, 3409, 3119, 4194, 2105, 279, 3869, 3572, 1085, 1071, 1673, 3108, 3576, 2118, 797, 3029, 2128, 3, 2715, 1878, 1673, 3974, 112, 2157, 92, 986, 3330, 11, 1860, 4, 272, 1041, 1176, 4283, 1357, 2070, 4332] payload_rows_per_dst=[15248, 15700, 14826, 15746, 15546, 13898, 14398, 14848] dup_rows_per_dst=[19116, 23062, 13694, 23918, 20920, 14784, 12218, 14222] combine_rows_per_src=[15248, 15700, 14826, 15746, 15546, 13898, 14398, 14848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120212 payload_rows_remote=105196 dup_rows=141932 dup_rows_remote=121066 dedup_ratio=0.541428 router_tokens_per_expert=[4009, 1532, 2572, 1780, 3425, 1, 2620, 1318, 1671, 5100, 3834, 1860, 3419, 2313, 1, 501, 857, 1949, 2302, 2493, 3, 1133, 4213, 2211, 481, 211, 4964, 1965, 4046, 3007, 3501, 2323, 203, 4218, 3434, 397, 1483, 1585, 3082, 1899, 1973, 753, 2209, 2368, 0, 2731, 2237, 1899, 4690, 168, 2254, 18, 1028, 4185, 5, 2488, 1, 287, 982, 1234, 4317, 1805, 1509, 4015] payload_rows_per_dst=[14446, 14852, 15398, 14964, 14148, 15016, 16086, 15302] dup_rows_per_dst=[16902, 12346, 19458, 15512, 13192, 20866, 24514, 19142] combine_rows_per_src=[14446, 14852, 15398, 14964, 14148, 15016, 16086, 15302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=119580 payload_rows_remote=104416 dup_rows=142564 dup_rows_remote=121832 dedup_ratio=0.543839 router_tokens_per_expert=[4714, 1233, 2398, 1590, 3073, 2, 2310, 2250, 2047, 5219, 4562, 2539, 2670, 3108, 3, 322, 1114, 1963, 1914, 2286, 11, 1187, 3659, 1428, 587, 229, 5194, 1631, 3756, 2401, 3507, 2001, 103, 3707, 3210, 456, 1641, 2092, 3574, 1982, 1411, 676, 2343, 3440, 7, 3636, 2304, 1888, 4329, 86, 2671, 66, 1084, 2966, 6, 1787, 4, 289, 982, 1643, 3612, 1807, 2292, 4070] payload_rows_per_dst=[15000, 15826, 14192, 15164, 15034, 14502, 15220, 14642] dup_rows_per_dst=[20140, 20134, 12932, 20732, 18496, 19624, 15750, 14756] combine_rows_per_src=[15000, 15826, 14192, 15164, 15034, 14502, 15220, 14642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=119276 payload_rows_remote=103600 dup_rows=142868 dup_rows_remote=121214 dedup_ratio=0.544998 router_tokens_per_expert=[4404, 1298, 2112, 1874, 2954, 2, 2782, 1287, 1735, 5366, 3703, 2478, 3085, 3164, 1, 487, 799, 1912, 1224, 1974, 4, 1160, 4232, 2187, 795, 407, 4696, 2050, 3779, 2429, 3658, 2175, 186, 5177, 3161, 752, 1302, 1714, 3090, 2479, 1642, 888, 2643, 2848, 1, 2882, 1991, 1497, 4554, 226, 2476, 51, 1047, 3226, 6, 2220, 1, 229, 1068, 1503, 4294, 1712, 1582, 4411] payload_rows_per_dst=[13106, 14402, 15694, 14126, 15260, 15196, 15816, 15676] dup_rows_per_dst=[11512, 11392, 21838, 13106, 19218, 22980, 21168, 21654] combine_rows_per_src=[13106, 14402, 15694, 14126, 15260, 15196, 15816, 15676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=119874 payload_rows_remote=103944 dup_rows=142270 dup_rows_remote=119524 dedup_ratio=0.542717 router_tokens_per_expert=[4235, 1315, 2472, 1918, 3031, 0, 2664, 1698, 2041, 4335, 4747, 1947, 3163, 2512, 0, 648, 784, 1955, 1820, 2443, 16, 1403, 4259, 2064, 820, 150, 4976, 1754, 3748, 2371, 4238, 2060, 144, 3976, 3578, 774, 1184, 1109, 2924, 1993, 1230, 897, 2483, 2408, 0, 2564, 1821, 2069, 3889, 145, 3641, 23, 1265, 3843, 5, 2406, 3, 257, 1143, 2134, 3742, 1876, 1594, 4365] payload_rows_per_dst=[13762, 15156, 15708, 14194, 14266, 15252, 15930, 15606] dup_rows_per_dst=[14012, 13614, 20956, 12496, 14472, 21644, 22746, 22330] combine_rows_per_src=[13762, 15156, 15708, 14194, 14266, 15252, 15930, 15606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=119756 payload_rows_remote=104072 dup_rows=142388 dup_rows_remote=121300 dedup_ratio=0.543167 router_tokens_per_expert=[4071, 1187, 2684, 1984, 3475, 0, 2768, 1535, 1608, 4482, 3554, 2501, 3251, 2304, 4, 682, 880, 2863, 1557, 2590, 1, 1247, 4321, 2059, 299, 296, 4973, 2017, 4273, 1924, 3412, 2267, 117, 5040, 3765, 617, 1420, 1744, 2984, 2219, 1791, 321, 2025, 3172, 7, 2652, 2383, 2133, 4493, 151, 1632, 12, 1208, 3492, 7, 1799, 9, 145, 1037, 1372, 4368, 2198, 1351, 4339] payload_rows_per_dst=[15256, 15684, 15064, 15426, 15372, 13582, 14450, 14922] dup_rows_per_dst=[21044, 21088, 15972, 22604, 20440, 15386, 11138, 14716] combine_rows_per_src=[15256, 15684, 15064, 15426, 15372, 13582, 14450, 14922] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[2, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=863305728 dispatch_meta_bytes=934496 combine_payload_bytes=863305728 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=864240224 ingress_bytes=1039466496 link_bytes=1039466496 token_bytes=1727545952 weight_movement_bytes=176160768 total_nvlink_bytes=1903706720 weight_to_token_ratio=0.101972 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.022727 gemm_active_groups=20 gemm_peak_group_rows=18048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.820578483e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=861765632 dispatch_meta_bytes=905048 combine_payload_bytes=861765632 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=862670680 ingress_bytes=1037926400 link_bytes=1037926400 token_bytes=1724436312 weight_movement_bytes=176160768 total_nvlink_bytes=1900597080 weight_to_token_ratio=0.102156 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=12544 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.811954588e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=855375872 dispatch_meta_bytes=904992 combine_payload_bytes=855375872 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=856280864 ingress_bytes=1383858176 link_bytes=1383858176 token_bytes=1711656736 weight_movement_bytes=528482304 total_nvlink_bytes=2240139040 weight_to_token_ratio=0.308755 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=14464 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.749028135e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=120838 payload_rows_remote=105870 dup_rows=141306 dup_rows_remote=124022 dedup_ratio=0.539040 router_tokens_per_expert=[3938, 1465, 2695, 1574, 2831, 4, 2216, 2097, 2345, 4565, 4302, 2314, 3051, 2767, 0, 841, 882, 2135, 2268, 2305, 12, 1411, 3711, 2117, 448, 155, 4837, 1574, 3499, 1909, 3750, 2256, 178, 4499, 3699, 544, 1509, 1420, 2944, 2809, 1693, 808, 2168, 2679, 9, 3110, 1876, 1105, 4273, 156, 2178, 134, 1211, 3797, 13, 1945, 3, 330, 1662, 1552, 4104, 2397, 1782, 4211] payload_rows_per_dst=[14796, 15416, 15322, 14764, 14968, 14490, 15766, 15316] dup_rows_per_dst=[18844, 16462, 17380, 15672, 17284, 17060, 20072, 18532] combine_rows_per_src=[14796, 15416, 15322, 14764, 14968, 14490, 15766, 15316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=848691200 dispatch_meta_bytes=899256 combine_payload_bytes=848691200 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=849590456 ingress_bytes=1024851968 link_bytes=1024851968 token_bytes=1698281656 weight_movement_bytes=176160768 total_nvlink_bytes=1874442424 weight_to_token_ratio=0.103729 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=17408 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.738743227e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=851509248 dispatch_meta_bytes=893872 combine_payload_bytes=851509248 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=852403120 ingress_bytes=1027670016 link_bytes=1027670016 token_bytes=1703912368 weight_movement_bytes=176160768 total_nvlink_bytes=1880073136 weight_to_token_ratio=0.103386 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=18048 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.754523119e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=852557824 dispatch_meta_bytes=901488 combine_payload_bytes=852557824 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=853459312 ingress_bytes=1381040128 link_bytes=1381040128 token_bytes=1706017136 weight_movement_bytes=528482304 total_nvlink_bytes=2234499440 weight_to_token_ratio=0.309775 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=15616 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.733248242e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=119938 payload_rows_remote=104398 dup_rows=142206 dup_rows_remote=118786 dedup_ratio=0.542473 router_tokens_per_expert=[3978, 1446, 3454, 2080, 2751, 1, 1687, 1486, 1247, 4355, 4131, 2250, 3014, 3578, 0, 746, 2006, 2590, 2566, 2256, 9, 1238, 3974, 2216, 264, 296, 5199, 2556, 2329, 2507, 2896, 2032, 167, 3343, 3893, 650, 2390, 1333, 3466, 2084, 1721, 891, 1990, 2276, 0, 2266, 2129, 2659, 4566, 132, 2407, 20, 909, 3162, 32, 2208, 17, 339, 1012, 1813, 4161, 2564, 1513, 3821] payload_rows_per_dst=[15540, 15642, 15284, 15344, 15238, 13536, 14492, 14862] dup_rows_per_dst=[23420, 21368, 18426, 18376, 18290, 14328, 12380, 15618] combine_rows_per_src=[15540, 15642, 15284, 15344, 15238, 13536, 14492, 14862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=867287040 dispatch_meta_bytes=919568 combine_payload_bytes=867287040 prefetch_bytes=88080384 reduce_bytes=264241152 egress_bytes=868206608 ingress_bytes=1219608576 link_bytes=1219608576 token_bytes=1735493648 weight_movement_bytes=352321536 total_nvlink_bytes=2087815184 weight_to_token_ratio=0.203009 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.022727 gemm_active_groups=19 gemm_peak_group_rows=15104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.829298936e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128698 payload_rows_remote=112318 dup_rows=133446 dup_rows_remote=122072 dedup_ratio=0.509056 router_tokens_per_expert=[0, 6, 1, 0, 8190, 3, 2411, 2, 8192, 7620, 1, 1647, 3443, 0, 0, 7342, 8180, 0, 5, 1, 0, 2, 8189, 1144, 1, 74, 1, 2097, 3, 2, 8188, 6362, 5, 1, 12, 0, 8056, 8, 0, 2234, 1620, 8164, 0, 2, 0, 1, 0, 8184, 5852, 15, 0, 597, 8185, 0, 8, 0, 27, 8189, 6734, 39, 0, 0, 0, 32] payload_rows_per_dst=[16384, 15746, 16384, 14668, 16368, 16380, 16384, 16384] dup_rows_per_dst=[21226, 9120, 18658, 10600, 19504, 11374, 29306, 13658] combine_rows_per_src=[16384, 15746, 16384, 14668, 16368, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130222 payload_rows_remote=113838 dup_rows=131922 dup_rows_remote=113786 dedup_ratio=0.503242 router_tokens_per_expert=[1, 6, 0, 1, 8179, 10, 697, 0, 8177, 7846, 0, 1660, 2526, 0, 11, 6657, 8180, 5, 7, 19, 10, 0, 8180, 859, 0, 487, 8, 3687, 1, 6, 8189, 7798, 5, 11, 2, 6, 8086, 8, 8, 5332, 811, 8177, 1, 8, 0, 0, 2, 8145, 3367, 0, 2, 454, 8180, 6, 10, 1, 9, 8176, 6928, 0, 6, 0, 3, 121] payload_rows_per_dst=[16378, 15554, 16384, 16384, 16380, 16384, 16382, 16376] dup_rows_per_dst=[17764, 10154, 18136, 23968, 22228, 17904, 7658, 14110] combine_rows_per_src=[16378, 15554, 16384, 16384, 16380, 16384, 16382, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130522 payload_rows_remote=114140 dup_rows=131622 dup_rows_remote=113572 dedup_ratio=0.502098 router_tokens_per_expert=[1, 0, 0, 2, 8189, 0, 2921, 1, 8187, 7376, 2, 2282, 2134, 0, 5, 7596, 8186, 0, 40, 2, 1, 0, 8188, 1547, 0, 148, 3, 2885, 2, 1, 8177, 6000, 31, 0, 2, 0, 7805, 1, 5, 3099, 2437, 8186, 3, 3, 8, 0, 0, 8166, 3344, 22, 1, 237, 8189, 3, 34, 3, 0, 8187, 7337, 4, 2, 0, 1, 86] payload_rows_per_dst=[16382, 15864, 16384, 16382, 16362, 16384, 16382, 16382] dup_rows_per_dst=[22220, 8174, 19544, 18050, 20276, 21222, 7284, 14852] combine_rows_per_src=[16382, 15864, 16384, 16382, 16362, 16384, 16382, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=2 dispatch_payload_bytes=855228416 dispatch_meta_bytes=892736 combine_payload_bytes=855228416 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=856121152 ingress_bytes=1207549952 link_bytes=1207549952 token_bytes=1711349568 weight_movement_bytes=352321536 total_nvlink_bytes=2063671104 weight_to_token_ratio=0.205874 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.021780 gemm_active_groups=20 gemm_peak_group_rows=13312 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.761775675e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125634 payload_rows_remote=109250 dup_rows=136510 dup_rows_remote=105760 dedup_ratio=0.520744 router_tokens_per_expert=[0, 0, 0, 1, 8191, 8, 2160, 0, 8167, 7405, 0, 1551, 1519, 0, 1, 7801, 8167, 0, 3, 1, 2, 0, 8190, 707, 4, 45, 0, 3126, 1, 11, 8191, 5490, 6, 0, 0, 0, 7784, 23, 0, 3505, 3331, 8186, 1, 1, 0, 0, 0, 8163, 5301, 14, 0, 409, 8190, 0, 39, 0, 0, 8186, 7087, 23, 0, 3, 0, 78] payload_rows_per_dst=[16384, 16384, 14892, 12456, 16376, 16374, 16384, 16384] dup_rows_per_dst=[4336, 21694, 1306, 5966, 21070, 23494, 27894, 30750] combine_rows_per_src=[16384, 16384, 14892, 12456, 16376, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129564 payload_rows_remote=113180 dup_rows=132580 dup_rows_remote=108342 dedup_ratio=0.505753 router_tokens_per_expert=[0, 2, 0, 1, 8191, 3, 943, 1, 8192, 7690, 0, 3401, 3516, 0, 9, 7292, 8183, 1, 6, 2, 0, 0, 8191, 1005, 0, 84, 0, 2031, 0, 20, 8188, 6356, 7, 0, 3, 1, 7938, 3, 0, 2937, 2932, 8158, 1, 0, 0, 0, 1, 8183, 3434, 4, 0, 493, 8190, 1, 11, 0, 10, 8191, 6914, 49, 1, 0, 1, 301] payload_rows_per_dst=[16384, 16276, 16384, 15030, 16338, 16384, 16384, 16384] dup_rows_per_dst=[6908, 23534, 17180, 8868, 20820, 14000, 24238, 17032] combine_rows_per_src=[16384, 16276, 16384, 15030, 16338, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130778 payload_rows_remote=114444 dup_rows=131366 dup_rows_remote=107822 dedup_ratio=0.501122 router_tokens_per_expert=[0, 0, 8, 0, 8188, 12, 1898, 0, 8191, 7639, 0, 1404, 3697, 0, 1, 7198, 8187, 3, 0, 5, 1, 3, 8180, 1233, 1, 104, 1, 3788, 2, 2, 8175, 6542, 8, 0, 4, 3, 7973, 17, 3, 1978, 1790, 8158, 3, 1, 4, 0, 3, 8184, 4915, 33, 1, 201, 8180, 3, 24, 1, 18, 8181, 6896, 0, 3, 1, 3, 20] payload_rows_per_dst=[16384, 16334, 16384, 16380, 16154, 16382, 16384, 16376] dup_rows_per_dst=[20210, 23544, 18840, 22358, 3818, 18396, 10332, 13868] combine_rows_per_src=[16384, 16334, 16384, 16380, 16154, 16382, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=920109056 dispatch_meta_bytes=937560 combine_payload_bytes=920109056 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=921046616 ingress_bytes=1360510976 link_bytes=1360510976 token_bytes=1841155672 weight_movement_bytes=440401920 total_nvlink_bytes=2281557592 weight_to_token_ratio=0.239199 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.618293560e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=932560896 dispatch_meta_bytes=910496 combine_payload_bytes=932560896 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=933471392 ingress_bytes=1284882432 link_bytes=1284882432 token_bytes=1866032288 weight_movement_bytes=352321536 total_nvlink_bytes=2218353824 weight_to_token_ratio=0.188808 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.194805283e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=935034880 dispatch_meta_bytes=910848 combine_payload_bytes=935034880 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=935945728 ingress_bytes=1375436800 link_bytes=1375436800 token_bytes=1870980608 weight_movement_bytes=440401920 total_nvlink_bytes=2311382528 weight_to_token_ratio=0.235386 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.018939 gemm_active_groups=20 gemm_peak_group_rows=26496 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.701871944e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=894976000 dispatch_meta_bytes=860040 combine_payload_bytes=894976000 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=895836040 ingress_bytes=1071136768 link_bytes=1071136768 token_bytes=1790812040 weight_movement_bytes=176160768 total_nvlink_bytes=1966972808 weight_to_token_ratio=0.098369 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.997918786e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=927170560 dispatch_meta_bytes=886088 combine_payload_bytes=927170560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=928056648 ingress_bytes=1103331328 link_bytes=1103331328 token_bytes=1855227208 weight_movement_bytes=176160768 total_nvlink_bytes=2031387976 weight_to_token_ratio=0.094954 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.020833 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.178194884e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130162 payload_rows_remote=113790 dup_rows=131982 dup_rows_remote=112174 dedup_ratio=0.503471 router_tokens_per_expert=[0, 2, 0, 0, 8192, 9, 1184, 0, 8179, 7535, 0, 918, 4089, 0, 0, 7230, 8189, 0, 14, 1, 0, 1, 8175, 1665, 6, 475, 1, 4110, 0, 5, 8184, 6652, 7, 0, 3, 0, 7951, 0, 2, 2592, 1535, 8184, 0, 1, 2, 0, 0, 8190, 4445, 1, 0, 369, 8191, 0, 33, 1, 1, 8188, 6410, 2, 0, 1, 0, 147] payload_rows_per_dst=[16384, 15750, 16384, 16122, 16372, 16384, 16384, 16382] dup_rows_per_dst=[18748, 8724, 19706, 20608, 19808, 19222, 12050, 13116] combine_rows_per_src=[16384, 15750, 16384, 16122, 16372, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=937525248 dispatch_meta_bytes=889064 combine_payload_bytes=937525248 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=938414312 ingress_bytes=1466007552 link_bytes=1466007552 token_bytes=1875939560 weight_movement_bytes=528482304 total_nvlink_bytes=2404421864 weight_to_token_ratio=0.281716 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.020833 gemm_active_groups=19 gemm_peak_group_rows=29184 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.209030350e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123510 payload_rows_remote=107126 dup_rows=138634 dup_rows_remote=118966 dedup_ratio=0.528847 router_tokens_per_expert=[1, 1, 0, 0, 8192, 0, 1651, 0, 8181, 7809, 2, 1101, 3488, 2, 0, 6091, 8152, 0, 6, 1, 1, 0, 8187, 1442, 1, 571, 2, 4248, 0, 5, 8183, 6665, 7, 3, 3, 0, 7386, 0, 0, 2736, 1688, 8183, 0, 1, 0, 0, 0, 7688, 5278, 2, 1, 863, 8187, 0, 18, 0, 26, 8189, 6790, 10, 0, 1, 0, 29] payload_rows_per_dst=[16384, 16382, 16382, 15338, 15792, 10464, 16384, 16384] dup_rows_per_dst=[19668, 28910, 19196, 28966, 4478, 11396, 12314, 13706] combine_rows_per_src=[16384, 16382, 16382, 15338, 15792, 10464, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=932167680 dispatch_meta_bytes=903856 combine_payload_bytes=932167680 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=933071536 ingress_bytes=1108328448 link_bytes=1108328448 token_bytes=1865239216 weight_movement_bytes=176160768 total_nvlink_bytes=2041399984 weight_to_token_ratio=0.094444 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=31744 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.206176670e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114664 dup_rows=131098 dup_rows_remote=115450 dedup_ratio=0.500099 router_tokens_per_expert=[0, 16, 0, 8047, 0, 1, 10, 0, 0, 0, 8186, 8189, 10, 0, 0, 8188, 0, 0, 7892, 0, 0, 8181, 1, 8187, 7, 3786, 84, 0, 8037, 0, 0, 1, 8187, 7569, 922, 3, 33, 6160, 4, 0, 0, 5, 0, 0, 0, 7816, 1, 5, 0, 8188, 0, 0, 0, 4, 0, 8168, 0, 0, 8191, 0, 6989, 0, 4, 0] payload_rows_per_dst=[16384, 16378, 16384, 16376, 16380, 16382, 16378, 16384] dup_rows_per_dst=[16138, 16392, 15764, 11480, 25350, 15648, 16342, 13984] combine_rows_per_src=[16384, 16378, 16384, 16376, 16380, 16382, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130980 payload_rows_remote=114598 dup_rows=131164 dup_rows_remote=115624 dedup_ratio=0.500351 router_tokens_per_expert=[6, 10, 5, 8154, 0, 3, 0, 0, 1, 6, 8170, 8179, 60, 0, 6, 8172, 2, 8, 7783, 3, 6, 8159, 0, 8177, 2, 3952, 290, 3, 8173, 0, 0, 1, 8185, 7313, 1403, 2, 5, 5357, 28, 6, 1, 8, 0, 0, 4, 7919, 0, 10, 6, 8185, 11, 6, 13, 3, 4, 8155, 2, 8, 8178, 74, 6855, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16380, 16320, 16362, 16384, 16384] dup_rows_per_dst=[16326, 16460, 15540, 24832, 11908, 15866, 16382, 13850] combine_rows_per_src=[16384, 16384, 16382, 16380, 16320, 16362, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114660 dup_rows=131100 dup_rows_remote=108572 dedup_ratio=0.500107 router_tokens_per_expert=[0, 3, 0, 8172, 0, 2, 12, 0, 0, 1, 8172, 8190, 4, 1, 0, 8191, 5, 0, 7957, 0, 0, 8182, 2, 8181, 9, 3071, 142, 0, 8046, 0, 0, 1, 8187, 7471, 1707, 4, 30, 6248, 14, 0, 0, 0, 0, 0, 0, 7900, 0, 7, 0, 8190, 0, 2, 4, 11, 0, 8185, 0, 1, 8189, 0, 6573, 0, 5, 0] payload_rows_per_dst=[16382, 16380, 16382, 16384, 16366, 16384, 16384, 16382] dup_rows_per_dst=[16358, 16356, 15910, 22528, 14582, 15812, 16400, 13154] combine_rows_per_src=[16382, 16380, 16382, 16384, 16366, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130910 payload_rows_remote=114528 dup_rows=131234 dup_rows_remote=101354 dedup_ratio=0.500618 router_tokens_per_expert=[0, 4, 0, 8091, 0, 20, 30, 0, 1, 0, 8102, 8187, 57, 0, 0, 8192, 1, 0, 7988, 0, 1, 8092, 0, 8151, 45, 2530, 191, 0, 7865, 0, 0, 0, 8186, 7723, 1278, 5, 13, 6506, 66, 0, 1, 8, 0, 1, 0, 7892, 0, 57, 1, 8191, 0, 0, 0, 11, 0, 8177, 0, 1, 8151, 0, 7255, 0, 1, 0] payload_rows_per_dst=[16382, 16384, 16328, 16282, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16210, 9648, 15836, 4980, 15724, 22580, 16376, 29880] combine_rows_per_src=[16382, 16384, 16328, 16282, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131022 payload_rows_remote=114638 dup_rows=131122 dup_rows_remote=114640 dedup_ratio=0.500191 router_tokens_per_expert=[1, 0, 1, 8107, 0, 0, 6, 0, 0, 1, 8174, 8191, 1, 0, 1, 8190, 0, 1, 8021, 1, 0, 8185, 0, 8191, 10, 2910, 210, 0, 8021, 1, 0, 0, 8180, 7728, 1241, 0, 66, 5993, 3, 1, 1, 0, 0, 0, 1, 7908, 0, 0, 1, 8190, 0, 1, 0, 3, 1, 8189, 1, 1, 8190, 1, 7144, 1, 2, 1] payload_rows_per_dst=[16384, 16384, 16384, 16336, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16228, 16352, 16032, 5968, 25310, 15818, 16482, 18932] combine_rows_per_src=[16384, 16384, 16384, 16336, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114670 dedup_ratio=0.500046 router_tokens_per_expert=[3, 10, 3, 8131, 0, 1, 0, 1, 0, 0, 8184, 8187, 24, 1, 3, 8182, 0, 4, 7904, 3, 3, 8156, 0, 8181, 15, 2328, 132, 2, 8155, 3, 0, 1, 8184, 7728, 956, 0, 39, 7003, 10, 3, 3, 0, 0, 0, 3, 7591, 2, 5, 3, 8188, 0, 3, 0, 4, 1, 8180, 2, 4, 8181, 1, 7356, 2, 0, 3] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16378] dup_rows_per_dst=[16276, 16414, 15756, 21256, 15100, 15188, 16374, 14720] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=2 dispatch_payload_bytes=877576192 dispatch_meta_bytes=904368 combine_payload_bytes=877576192 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=878480560 ingress_bytes=1053736960 link_bytes=1053736960 token_bytes=1756056752 weight_movement_bytes=176160768 total_nvlink_bytes=1932217520 weight_to_token_ratio=0.100316 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.900487125e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=920456 combine_payload_bytes=939327488 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940247944 ingress_bytes=1291649024 link_bytes=1291649024 token_bytes=1879575432 weight_movement_bytes=352321536 total_nvlink_bytes=2231896968 weight_to_token_ratio=0.187447 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232695374e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939294720 dispatch_meta_bytes=892928 combine_payload_bytes=939294720 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940187648 ingress_bytes=1115455488 link_bytes=1115455488 token_bytes=1879482368 weight_movement_bytes=176160768 total_nvlink_bytes=2055643136 weight_to_token_ratio=0.093728 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246085119e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=938786816 dispatch_meta_bytes=920888 combine_payload_bytes=938786816 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939707704 ingress_bytes=1467269120 link_bytes=1467269120 token_bytes=1878494520 weight_movement_bytes=528482304 total_nvlink_bytes=2406976824 weight_to_token_ratio=0.281333 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.017992 gemm_active_groups=18 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.216094604e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=938213376 dispatch_meta_bytes=863528 combine_payload_bytes=938213376 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939076904 ingress_bytes=1114374144 link_bytes=1114374144 token_bytes=1877290280 weight_movement_bytes=176160768 total_nvlink_bytes=2053451048 weight_to_token_ratio=0.093838 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.240030044e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 4, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130948 payload_rows_remote=114682 dup_rows=131196 dup_rows_remote=112156 dedup_ratio=0.500473 router_tokens_per_expert=[0, 18, 0, 8141, 0, 4, 4, 0, 0, 0, 8156, 8176, 46, 0, 0, 8191, 3, 0, 8009, 0, 0, 8155, 2, 8188, 8, 2777, 270, 0, 8122, 0, 1, 0, 8190, 7556, 1250, 1, 23, 5859, 5, 0, 0, 0, 0, 0, 0, 7952, 0, 6, 0, 8191, 1, 1, 1, 4, 0, 8186, 0, 1, 8188, 0, 7386, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16380, 16384, 16266, 16384, 16384, 16384] dup_rows_per_dst=[16326, 16374, 15958, 16434, 19040, 15914, 16384, 14766] combine_rows_per_src=[16384, 16382, 16380, 16384, 16266, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=917112 combine_payload_bytes=939114496 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940031608 ingress_bytes=1115275264 link_bytes=1115275264 token_bytes=1879146104 weight_movement_bytes=176160768 total_nvlink_bytes=2055306872 weight_to_token_ratio=0.093745 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245075940e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917384 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940343176 ingress_bytes=1467908096 link_bytes=1467908096 token_bytes=1879768968 weight_movement_bytes=528482304 total_nvlink_bytes=2408251272 weight_to_token_ratio=0.281142 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.018939 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219672603e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 2, 2, 0, 0, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=113788 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 0, 8189, 0, 0, 27, 1042, 8124, 0, 0, 0, 259, 0, 7315, 8187, 0, 7452, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8189, 0, 8192, 8192, 0, 8188, 8183, 0, 459, 0, 0, 0, 9, 8192, 1, 0, 7812, 287, 0, 5, 0, 8187, 0, 8192, 0, 3, 0, 3, 0] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16432, 17096, 14902, 16378, 16384, 17294, 16210, 16386] combine_rows_per_src=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131054 payload_rows_remote=114670 dup_rows=131090 dup_rows_remote=114718 dedup_ratio=0.500069 router_tokens_per_expert=[0, 0, 0, 0, 8191, 0, 0, 190, 1656, 8090, 0, 0, 0, 114, 0, 7266, 8181, 0, 6628, 8192, 0, 0, 0, 8186, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8187, 0, 8192, 8192, 0, 8189, 8186, 0, 1472, 0, 0, 0, 11, 8192, 2, 0, 7136, 229, 0, 5, 0, 8183, 0, 8188, 0, 2, 0, 5, 0] payload_rows_per_dst=[16384, 16380, 16370, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16762, 17872, 13252, 16372, 16384, 19332, 14744, 16372] combine_rows_per_src=[16384, 16380, 16370, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=907352 combine_payload_bytes=939474944 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=940382296 ingress_bytes=1820278784 link_bytes=1820278784 token_bytes=1879857240 weight_movement_bytes=880803840 total_nvlink_bytes=2760661080 weight_to_token_ratio=0.468548 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=25344 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019280137e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114686 dup_rows=131078 dup_rows_remote=118484 dedup_ratio=0.500023 router_tokens_per_expert=[6, 9, 0, 0, 8092, 11, 5, 171, 1998, 8048, 0, 7, 0, 260, 0, 7374, 8179, 0, 6280, 8182, 9, 8, 6, 8183, 0, 7, 0, 18, 0, 7, 3, 2, 1, 8, 35, 8182, 7, 8178, 8180, 0, 8181, 8180, 8, 963, 0, 0, 0, 13, 8184, 0, 8, 7532, 163, 8, 2, 0, 8179, 0, 8181, 4, 1, 1, 1, 7] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16564, 18990, 12594, 16410, 16438, 18306, 15410, 16366] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130892 payload_rows_remote=114508 dup_rows=131252 dup_rows_remote=114790 dedup_ratio=0.500687 router_tokens_per_expert=[1, 0, 0, 0, 8140, 0, 0, 132, 2734, 7817, 0, 1, 0, 79, 1, 6811, 8188, 0, 7160, 8191, 1, 1, 1, 8183, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 8192, 0, 8190, 8191, 0, 8192, 8187, 1, 1055, 0, 1, 0, 43, 8191, 0, 1, 6072, 928, 1, 0, 0, 8191, 0, 8190, 1, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16206, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15062, 6592, 21822, 16390, 15666, 19622, 19636, 16462] combine_rows_per_src=[16384, 16206, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126078 payload_rows_remote=109706 dup_rows=136066 dup_rows_remote=118840 dedup_ratio=0.519051 router_tokens_per_expert=[0, 3, 0, 7606, 1, 5, 1, 0, 0, 0, 8139, 8180, 485, 0, 0, 8184, 0, 0, 8072, 0, 0, 8164, 8, 8181, 1, 3718, 77, 0, 8094, 0, 1, 1, 8178, 7939, 1201, 11, 21, 7189, 15, 0, 0, 20, 0, 1, 0, 7112, 0, 11, 0, 8190, 0, 0, 0, 10, 0, 8163, 0, 0, 8181, 0, 5906, 0, 3, 0] payload_rows_per_dst=[16372, 16382, 16374, 16384, 11420, 16384, 16384, 16378] dup_rows_per_dst=[17226, 23090, 20350, 23756, 9232, 14272, 16338, 11802] combine_rows_per_src=[16372, 16382, 16374, 16384, 11420, 16384, 16384, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=10 unused_prefetch_slots=22 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=5 remote_demand_peak=3 slot_utilization=0.312500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 10, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 5, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131024 payload_rows_remote=114640 dup_rows=131120 dup_rows_remote=115526 dedup_ratio=0.500183 router_tokens_per_expert=[1, 1, 0, 1, 8177, 1, 0, 49, 1421, 8189, 0, 1, 0, 165, 10, 7290, 8188, 0, 7660, 8191, 1, 1, 0, 8191, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 8191, 0, 8191, 8190, 0, 8189, 8191, 1, 217, 1, 0, 0, 1, 8191, 0, 1, 7494, 301, 0, 2, 0, 8172, 2, 8191, 2, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16338, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16458, 14230, 18906, 16382, 16382, 16816, 15594, 16352] combine_rows_per_src=[16384, 16338, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=113802 dedup_ratio=0.500023 router_tokens_per_expert=[3, 4, 0, 3, 8156, 3, 1, 192, 1678, 8108, 0, 3, 0, 243, 1, 6797, 8185, 0, 7220, 8188, 3, 3, 3, 8187, 0, 0, 2, 0, 4, 0, 3, 2, 0, 3, 4, 8189, 7, 8188, 8187, 0, 8188, 8189, 4, 376, 2, 0, 1, 4, 8188, 0, 3, 7629, 533, 0, 5, 0, 8183, 1, 8188, 4, 1, 0, 0, 3] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16714, 17276, 14452, 16384, 16398, 17144, 16334, 16376] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=913864 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940356040 ingress_bytes=1379844096 link_bytes=1379844096 token_bytes=1879798216 weight_movement_bytes=440401920 total_nvlink_bytes=2320200136 weight_to_token_ratio=0.234281 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726550962e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=917552 combine_payload_bytes=939376640 prefetch_bytes=440401920 reduce_bytes=0 egress_bytes=940294192 ingress_bytes=1379778560 link_bytes=1379778560 token_bytes=1879670832 weight_movement_bytes=440401920 total_nvlink_bytes=2320072752 weight_to_token_ratio=0.234297 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726183988e-05 prefetch_bytes_per_gemm_flop=2.466066919e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=932680 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=440401920 egress_bytes=940440392 ingress_bytes=1556070400 link_bytes=1556070400 token_bytes=1879948104 weight_movement_bytes=616562688 total_nvlink_bytes=2496510792 weight_to_token_ratio=0.327968 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.713344704e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=938049536 dispatch_meta_bytes=917192 combine_payload_bytes=938049536 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938966728 ingress_bytes=1114210304 link_bytes=1114210304 token_bytes=1877016264 weight_movement_bytes=176160768 total_nvlink_bytes=2053177032 weight_to_token_ratio=0.093851 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.239112608e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=2 dispatch_payload_bytes=898711552 dispatch_meta_bytes=914184 combine_payload_bytes=898711552 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=899625736 ingress_bytes=1251033088 link_bytes=1251033088 token_bytes=1798337288 weight_movement_bytes=352321536 total_nvlink_bytes=2150658824 weight_to_token_ratio=0.195915 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.005263086e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939130880 dispatch_meta_bytes=920664 combine_payload_bytes=939130880 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940051544 ingress_bytes=1115291648 link_bytes=1115291648 token_bytes=1879182424 weight_movement_bytes=176160768 total_nvlink_bytes=2055343192 weight_to_token_ratio=0.093743 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245167683e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=913936 combine_payload_bytes=939474944 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940388880 ingress_bytes=1379876864 link_bytes=1379876864 token_bytes=1879863824 weight_movement_bytes=440401920 total_nvlink_bytes=2320265744 weight_to_token_ratio=0.234273 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.726734449e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114696 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 100, 1244, 8080, 0, 0, 0, 82, 0, 7690, 8191, 0, 7069, 8192, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 8192, 8192, 0, 8184, 8192, 0, 181, 0, 1, 0, 30, 8192, 0, 0, 7961, 338, 0, 1, 0, 8191, 0, 8192, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16584, 17812, 14138, 16386, 16384, 16792, 16600, 16384] combine_rows_per_src=[16384, 16380, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=111340 dedup_ratio=0.500038 router_tokens_per_expert=[0, 35, 1, 0, 7768, 0, 0, 0, 0, 8091, 5, 0, 0, 0, 0, 8182, 17, 0, 0, 2106, 0, 0, 5183, 0, 8186, 0, 7, 8192, 1, 8186, 0, 8169, 22, 3, 30, 3, 0, 8190, 0, 0, 8191, 6, 1693, 0, 0, 0, 0, 8173, 8192, 8192, 0, 0, 0, 0, 12, 0, 1, 0, 8187, 7857, 0, 0, 8191, 0] payload_rows_per_dst=[16384, 16374, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15604, 16182, 14598, 16352, 16482, 19742, 16408, 15714] combine_rows_per_src=[16384, 16374, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917504 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940376064 ingress_bytes=1467940864 link_bytes=1467940864 token_bytes=1879834624 weight_movement_bytes=528482304 total_nvlink_bytes=2408316928 weight_to_token_ratio=0.281132 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219856090e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=114814 dedup_ratio=0.500053 router_tokens_per_expert=[0, 160, 23, 0, 7781, 0, 0, 0, 4, 8008, 7, 0, 0, 0, 0, 8183, 11, 0, 0, 2801, 0, 0, 5069, 0, 8185, 0, 6, 8192, 1, 8185, 0, 8134, 16, 6, 33, 0, 0, 8190, 2, 0, 8187, 1, 2084, 0, 2, 0, 0, 8167, 8170, 8192, 0, 0, 0, 0, 31, 0, 0, 0, 8186, 6863, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16376, 16378, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15922, 16028, 15756, 16272, 16482, 20498, 16402, 13726] combine_rows_per_src=[16384, 16376, 16378, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=118134 dedup_ratio=0.500038 router_tokens_per_expert=[12, 191, 5, 1, 7017, 0, 9, 2, 203, 8039, 3, 8, 8, 8, 0, 8179, 3, 8, 0, 1285, 2, 7, 5179, 0, 8184, 11, 15, 8183, 1, 8181, 0, 7956, 40, 1, 25, 1, 3, 8183, 5, 9, 8163, 17, 4507, 8, 0, 7, 4, 8134, 8183, 8171, 8, 0, 0, 0, 8, 4, 0, 8, 8182, 6516, 6, 7, 8164, 8] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16380, 16384, 16380] dup_rows_per_dst=[14456, 16514, 12948, 15950, 16512, 25300, 16364, 13038] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16380, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130610 payload_rows_remote=114226 dup_rows=131534 dup_rows_remote=100692 dedup_ratio=0.501762 router_tokens_per_expert=[0, 350, 0, 1, 8001, 0, 1, 0, 0, 7950, 5, 0, 1, 1, 0, 8184, 5, 0, 1, 1162, 0, 1, 5443, 1, 8191, 1, 7, 8177, 0, 8180, 0, 8088, 59, 3, 144, 3, 1, 8191, 0, 1, 8190, 0, 3089, 1, 0, 1, 1, 8179, 8173, 8191, 1, 0, 0, 0, 69, 0, 0, 0, 8187, 6645, 0, 1, 8190, 1] payload_rows_per_dst=[16384, 16372, 15940, 16382, 16384, 16380, 16384, 16384] dup_rows_per_dst=[16704, 15910, 11324, 16154, 16144, 7972, 16484, 30842] combine_rows_per_src=[16384, 16372, 15940, 16382, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[4, 0, 2, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114732 dedup_ratio=0.500023 router_tokens_per_expert=[0, 105, 2, 0, 7506, 0, 1, 1, 10, 8090, 2, 1, 1, 1, 0, 8191, 3, 1, 0, 3540, 0, 1, 4798, 1, 8190, 0, 3, 8191, 0, 8191, 0, 8164, 7, 0, 1, 0, 1, 8191, 0, 0, 8188, 2, 1366, 1, 0, 1, 1, 8181, 8173, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 8191, 7388, 1, 1, 8190, 1] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16382, 16382, 16384, 16384] dup_rows_per_dst=[15226, 16208, 16686, 16334, 16400, 17630, 16346, 16248] combine_rows_per_src=[16384, 16384, 16384, 16382, 16382, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130800 payload_rows_remote=114416 dup_rows=131344 dup_rows_remote=114848 dedup_ratio=0.501038 router_tokens_per_expert=[0, 0, 0, 0, 8191, 0, 0, 57, 2074, 7746, 0, 0, 0, 113, 0, 6555, 8188, 0, 7555, 8192, 0, 0, 0, 8180, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8189, 0, 8192, 8192, 0, 8192, 8177, 0, 409, 0, 0, 0, 21, 8192, 0, 0, 8046, 223, 0, 4, 0, 8187, 0, 8191, 0, 0, 0, 2, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16116, 16384, 16384] dup_rows_per_dst=[16496, 21436, 21050, 16370, 17036, 6034, 16546, 16376] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16116, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=904072 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940346248 ingress_bytes=1644085248 link_bytes=1644085248 token_bytes=1879788424 weight_movement_bytes=704643072 total_nvlink_bytes=2584431496 weight_to_token_ratio=0.374852 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=29184 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206191114e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131010 payload_rows_remote=114652 dup_rows=131134 dup_rows_remote=115254 dedup_ratio=0.500237 router_tokens_per_expert=[1, 129, 1, 0, 7719, 0, 4, 3, 2, 7932, 5, 3, 4, 3, 0, 8170, 6, 2, 0, 2385, 0, 3, 6419, 2, 8188, 2, 2, 8186, 0, 8188, 1, 8152, 63, 0, 6, 0, 8, 8188, 1, 1, 8186, 1, 1019, 4, 0, 3, 4, 8171, 8188, 8188, 3, 0, 0, 0, 5, 1, 0, 3, 8189, 7131, 3, 3, 8188, 3] payload_rows_per_dst=[16384, 16358, 16384, 16380, 16380, 16356, 16384, 16384] dup_rows_per_dst=[15702, 15880, 17626, 16368, 16532, 18362, 16386, 14278] combine_rows_per_src=[16384, 16358, 16384, 16380, 16380, 16356, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=917952 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=940327360 ingress_bytes=1732132864 link_bytes=1732132864 token_bytes=1879736768 weight_movement_bytes=792723456 total_nvlink_bytes=2672460224 weight_to_token_ratio=0.421720 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699221011e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=931248 combine_payload_bytes=939442176 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940373424 ingress_bytes=1203683328 link_bytes=1203683328 token_bytes=1879815600 weight_movement_bytes=264241152 total_nvlink_bytes=2144056752 weight_to_token_ratio=0.140568 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740124195e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=935739392 dispatch_meta_bytes=859672 combine_payload_bytes=935739392 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=936599064 ingress_bytes=1288060928 link_bytes=1288060928 token_bytes=1872338456 weight_movement_bytes=352321536 total_nvlink_bytes=2224659992 weight_to_token_ratio=0.188172 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.212603534e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=917656 combine_payload_bytes=939474944 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940392600 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879867544 weight_movement_bytes=176160768 total_nvlink_bytes=2056028312 weight_to_token_ratio=0.093709 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025568 gemm_active_groups=11 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=2 dispatch_payload_bytes=937295872 dispatch_meta_bytes=917056 combine_payload_bytes=937295872 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938212928 ingress_bytes=1113456640 link_bytes=1113456640 token_bytes=1875508800 weight_movement_bytes=176160768 total_nvlink_bytes=2051669568 weight_to_token_ratio=0.093927 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.017045 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.234892404e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939229184 dispatch_meta_bytes=919624 combine_payload_bytes=939229184 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940148808 ingress_bytes=1115389952 link_bytes=1115389952 token_bytes=1879377992 weight_movement_bytes=176160768 total_nvlink_bytes=2055538760 weight_to_token_ratio=0.093734 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245718145e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114480 dedup_ratio=0.500038 router_tokens_per_expert=[0, 38, 1, 0, 7121, 0, 0, 0, 42, 8049, 3, 0, 0, 0, 0, 8189, 13, 0, 0, 2835, 0, 0, 6414, 0, 8191, 0, 4, 8192, 1, 8187, 0, 8170, 23, 1, 86, 0, 2, 8191, 0, 0, 8192, 1, 1292, 0, 1, 0, 0, 8177, 8191, 8191, 0, 0, 0, 0, 17, 0, 0, 0, 8191, 6874, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16382, 16382, 16378, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14320, 16184, 18522, 16350, 16602, 18942, 16414, 13748] combine_rows_per_src=[16384, 16382, 16382, 16378, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131020 payload_rows_remote=114648 dup_rows=131124 dup_rows_remote=107886 dedup_ratio=0.500198 router_tokens_per_expert=[0, 0, 8192, 8162, 7, 0, 45, 0, 0, 0, 0, 0, 2297, 0, 5525, 1021, 8192, 0, 0, 0, 0, 0, 1, 7493, 8192, 0, 245, 0, 0, 0, 0, 8152, 3, 0, 8187, 0, 8163, 8192, 1373, 8192, 0, 0, 7677, 848, 4789, 0, 8191, 0, 0, 0, 0, 4016, 0, 150, 0, 0, 5477, 0, 8174, 14, 0, 55, 0, 47] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16344, 16372, 16384, 16384] dup_rows_per_dst=[16428, 17680, 14988, 16794, 2734, 23238, 24712, 14550] combine_rows_per_src=[16384, 16384, 16384, 16384, 16344, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131048 payload_rows_remote=114664 dup_rows=131096 dup_rows_remote=114462 dedup_ratio=0.500092 router_tokens_per_expert=[0, 0, 8192, 8114, 14, 0, 14, 0, 0, 0, 1, 0, 1716, 0, 5387, 2394, 8190, 0, 0, 0, 1, 0, 2, 6733, 8192, 0, 168, 0, 0, 0, 0, 8149, 2, 0, 8180, 0, 7667, 8192, 1998, 8189, 0, 0, 8087, 90, 4713, 0, 8192, 0, 0, 0, 0, 4218, 0, 160, 1, 0, 5809, 0, 8171, 34, 0, 68, 0, 34] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16360, 16384, 16384] dup_rows_per_dst=[16284, 18970, 13468, 16634, 22114, 23020, 8758, 11848] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16360, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=916632 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940358808 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879800984 weight_movement_bytes=176160768 total_nvlink_bytes=2055961752 weight_to_token_ratio=0.093712 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114658 dup_rows=131102 dup_rows_remote=117120 dedup_ratio=0.500114 router_tokens_per_expert=[3, 6, 8182, 8163, 4, 8, 38, 2, 3, 0, 23, 9, 1016, 0, 5802, 785, 8178, 10, 1, 0, 0, 5, 1, 6988, 8183, 10, 307, 0, 9, 1, 7, 8176, 0, 9, 8176, 5, 7485, 8131, 1954, 8166, 9, 1, 7902, 185, 4998, 12, 8182, 15, 13, 6, 0, 3888, 0, 649, 2, 9, 7089, 9, 8156, 28, 1, 44, 0, 28] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16358, 16382, 16384] dup_rows_per_dst=[16428, 15258, 13982, 17002, 30730, 14294, 9082, 14326] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16358, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 2, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127058 payload_rows_remote=110674 dup_rows=135086 dup_rows_remote=105074 dedup_ratio=0.515312 router_tokens_per_expert=[0, 1, 8191, 8163, 7, 1, 109, 0, 1, 0, 0, 1, 1112, 1, 4115, 1500, 8186, 1, 0, 0, 0, 0, 0, 6745, 8191, 1, 340, 0, 1, 0, 0, 8093, 1, 0, 8175, 0, 7910, 8186, 676, 8191, 1, 0, 7690, 143, 6174, 1, 8191, 1, 1, 1, 0, 5429, 0, 111, 0, 1, 7022, 1, 8141, 135, 0, 105, 0, 25] payload_rows_per_dst=[16382, 12914, 16374, 16384, 15862, 16374, 16384, 16384] dup_rows_per_dst=[16108, 8772, 13490, 11426, 9466, 18386, 27426, 30012] combine_rows_per_src=[16382, 12914, 16374, 16384, 15862, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130766 payload_rows_remote=114382 dup_rows=131378 dup_rows_remote=107290 dedup_ratio=0.501167 router_tokens_per_expert=[0, 1, 8191, 8164, 2, 1, 51, 0, 0, 0, 5, 1, 1074, 1, 6749, 1040, 8189, 1, 0, 0, 0, 0, 0, 7496, 8191, 0, 119, 1, 1, 1, 1, 8180, 0, 0, 8184, 1, 8090, 8190, 2701, 8189, 1, 1, 7968, 136, 4063, 1, 8190, 0, 0, 1, 0, 3468, 1, 390, 1, 1, 5796, 1, 8176, 46, 0, 0, 0, 16] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16234, 16230, 16384, 16384] dup_rows_per_dst=[16436, 17734, 14990, 16604, 5350, 8242, 24088, 27934] combine_rows_per_src=[16384, 16384, 16382, 16384, 16234, 16230, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939196416 dispatch_meta_bytes=890136 combine_payload_bytes=939196416 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940086552 ingress_bytes=1643839488 link_bytes=1643839488 token_bytes=1879282968 weight_movement_bytes=704643072 total_nvlink_bytes=2583926040 weight_to_token_ratio=0.374953 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.019886 gemm_active_groups=20 gemm_peak_group_rows=23552 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.204814960e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939327488 dispatch_meta_bytes=916504 combine_payload_bytes=939327488 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940243992 ingress_bytes=1291649024 link_bytes=1291649024 token_bytes=1879571480 weight_movement_bytes=352321536 total_nvlink_bytes=2231893016 weight_to_token_ratio=0.187448 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232695374e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130640 payload_rows_remote=114256 dup_rows=131504 dup_rows_remote=108366 dedup_ratio=0.501648 router_tokens_per_expert=[0, 183, 1, 6, 5726, 0, 0, 0, 1, 8121, 10, 0, 0, 0, 0, 8175, 31, 0, 0, 3776, 0, 0, 6640, 0, 8189, 0, 4, 8064, 1, 8181, 0, 8166, 18, 3, 162, 0, 1, 8192, 0, 0, 8192, 0, 1540, 0, 0, 0, 0, 8158, 8189, 8191, 0, 0, 0, 0, 29, 0, 2, 0, 8189, 6751, 0, 0, 8180, 0] payload_rows_per_dst=[16384, 16372, 16384, 16384, 16384, 15964, 16384, 16384] dup_rows_per_dst=[23138, 18238, 24064, 17306, 15924, 3084, 16268, 13482] combine_rows_per_src=[16384, 16372, 16384, 16384, 16384, 15964, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131004 payload_rows_remote=114620 dup_rows=131140 dup_rows_remote=115004 dedup_ratio=0.500259 router_tokens_per_expert=[2, 3, 8188, 8171, 0, 3, 146, 0, 0, 0, 0, 4, 915, 2, 6220, 937, 8180, 4, 3, 0, 0, 2, 0, 7129, 8188, 4, 318, 0, 4, 2, 3, 8155, 0, 1, 8169, 3, 8067, 8186, 1946, 8182, 4, 2, 7756, 291, 4713, 4, 8188, 3, 2, 3, 0, 4571, 1, 119, 1, 4, 5998, 4, 8172, 45, 0, 10, 0, 44] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16370, 16338, 16384, 16384] dup_rows_per_dst=[16642, 16136, 15442, 16964, 34508, 9900, 9386, 12162] combine_rows_per_src=[16384, 16384, 16376, 16384, 16370, 16338, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=906641408 dispatch_meta_bytes=862992 combine_payload_bytes=906641408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=907504400 ingress_bytes=1082802176 link_bytes=1082802176 token_bytes=1814145808 weight_movement_bytes=176160768 total_nvlink_bytes=1990306576 weight_to_token_ratio=0.097104 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.023674 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.063240202e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=927112 combine_payload_bytes=939278336 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940205448 ingress_bytes=1115439104 link_bytes=1115439104 token_bytes=1879483784 weight_movement_bytes=176160768 total_nvlink_bytes=2055644552 weight_to_token_ratio=0.093728 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245993375e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=937017344 dispatch_meta_bytes=886688 combine_payload_bytes=937017344 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=937904032 ingress_bytes=1289338880 link_bytes=1289338880 token_bytes=1874921376 weight_movement_bytes=352321536 total_nvlink_bytes=2227242912 weight_to_token_ratio=0.187913 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.219759531e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130378 payload_rows_remote=114688 dup_rows=131766 dup_rows_remote=123918 dedup_ratio=0.502647 router_tokens_per_expert=[0, 0, 8192, 8171, 1, 0, 5, 0, 0, 0, 0, 0, 1745, 0, 5617, 1402, 8192, 0, 0, 0, 0, 0, 1, 7514, 8192, 0, 864, 0, 0, 0, 0, 8150, 0, 0, 8191, 0, 6963, 8192, 2219, 8192, 0, 0, 7728, 521, 4399, 0, 8192, 0, 0, 0, 0, 4620, 0, 159, 0, 0, 5361, 0, 8163, 16, 1, 99, 0, 10] payload_rows_per_dst=[16384, 16384, 16384, 16384, 15690, 16384, 16384, 16384] dup_rows_per_dst=[16354, 17528, 15030, 18028, 7848, 25296, 20766, 10916] combine_rows_per_src=[16384, 16384, 16384, 16384, 15690, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=2 dispatch_payload_bytes=935985152 dispatch_meta_bytes=890488 combine_payload_bytes=935985152 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936875640 ingress_bytes=1288306688 link_bytes=1288306688 token_bytes=1872860792 weight_movement_bytes=352321536 total_nvlink_bytes=2225182328 weight_to_token_ratio=0.188119 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.213979687e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=938967040 dispatch_meta_bytes=918496 combine_payload_bytes=938967040 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939885536 ingress_bytes=1115127808 link_bytes=1115127808 token_bytes=1878852576 weight_movement_bytes=176160768 total_nvlink_bytes=2055013344 weight_to_token_ratio=0.093760 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244250248e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=115476 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 0, 12, 0, 1, 0, 7780, 0, 8192, 8164, 0, 8182, 11, 0, 20, 0, 0, 11, 8190, 0, 165, 0, 0, 8130, 8192, 0, 0, 8189, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 7800, 0, 8192, 8192, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 1271, 8192, 0, 8192] payload_rows_per_dst=[16384, 16380, 16384, 16378, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16354, 15566, 16772, 16260, 16290, 15606, 15308, 18926] combine_rows_per_src=[16384, 16380, 16384, 16378, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=954424 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940478520 ingress_bytes=1644167168 link_bytes=1644167168 token_bytes=1880002616 weight_movement_bytes=704643072 total_nvlink_bytes=2584645688 weight_to_token_ratio=0.374810 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=31104 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206649832e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114684 dup_rows=131092 dup_rows_remote=114758 dedup_ratio=0.500076 router_tokens_per_expert=[0, 0, 0, 0, 23, 0, 2, 0, 7026, 0, 8192, 8161, 0, 8172, 31, 0, 49, 0, 0, 20, 8186, 0, 363, 0, 0, 8169, 8192, 0, 0, 8182, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 8192, 9, 2, 0, 0, 7491, 0, 8192, 8192, 0, 0, 0, 7644, 0, 0, 0, 0, 0, 0, 2108, 8192, 0, 8192] payload_rows_per_dst=[16384, 16380, 16384, 16368, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16372, 14078, 17236, 16334, 16180, 15004, 15288, 20600] combine_rows_per_src=[16384, 16380, 16384, 16368, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130966 payload_rows_remote=114582 dup_rows=131178 dup_rows_remote=111948 dedup_ratio=0.500404 router_tokens_per_expert=[1, 0, 1, 1, 127, 0, 0, 0, 7382, 0, 8191, 8179, 1, 8107, 33, 0, 0, 0, 1, 10, 8189, 0, 375, 0, 0, 8181, 8187, 0, 1, 8183, 1, 0, 1, 0, 8176, 1, 1, 0, 1, 0, 1, 8191, 1, 0, 1, 0, 7349, 1, 8191, 8188, 0, 1, 0, 8010, 0, 0, 0, 1, 0, 1, 1423, 8191, 0, 8191] payload_rows_per_dst=[16384, 16292, 16384, 16370, 16384, 16384, 16384, 16384] dup_rows_per_dst=[15670, 15700, 10444, 15324, 17388, 14704, 22718, 19230] combine_rows_per_src=[16384, 16292, 16384, 16370, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114662 dup_rows=131106 dup_rows_remote=113908 dedup_ratio=0.500130 router_tokens_per_expert=[11, 0, 146, 9, 58, 11, 1, 18, 7905, 13, 8181, 8174, 13, 8165, 7, 8, 88, 0, 9, 9, 8177, 1, 332, 0, 0, 8154, 8179, 5, 6, 8172, 9, 0, 7, 0, 8155, 11, 9, 0, 1, 0, 7, 8182, 2, 0, 10, 0, 7547, 9, 8171, 8173, 3, 8, 0, 7480, 0, 8, 0, 8, 6, 0, 1066, 8181, 0, 8167] payload_rows_per_dst=[16384, 16376, 16376, 16382, 16380, 16384, 16378, 16378] dup_rows_per_dst=[16834, 15846, 17198, 16310, 16344, 15130, 14966, 18478] combine_rows_per_src=[16384, 16376, 16376, 16382, 16380, 16384, 16378, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114654 dup_rows=131106 dup_rows_remote=115764 dedup_ratio=0.500130 router_tokens_per_expert=[1, 0, 12, 1, 25, 0, 0, 0, 6764, 3, 8191, 8182, 1, 8173, 0, 1, 12, 0, 1, 0, 8191, 0, 719, 0, 0, 8173, 8191, 0, 1, 8189, 1, 0, 1, 0, 8169, 1, 1, 0, 0, 0, 1, 8191, 0, 2, 1, 0, 7595, 1, 8190, 8191, 0, 1, 0, 7671, 0, 0, 0, 1, 1, 1, 1839, 8191, 0, 8191] payload_rows_per_dst=[16384, 16354, 16384, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16440, 13530, 17842, 16348, 16342, 15198, 15342, 20064] combine_rows_per_src=[16384, 16354, 16384, 16380, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=920616 combine_payload_bytes=939442176 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940362792 ingress_bytes=1115602944 link_bytes=1115602944 token_bytes=1879804968 weight_movement_bytes=176160768 total_nvlink_bytes=2055965736 weight_to_token_ratio=0.093712 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246910811e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 2, 0, 2, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917768 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940409096 ingress_bytes=1467973632 link_bytes=1467973632 token_bytes=1879900424 weight_movement_bytes=528482304 total_nvlink_bytes=2408382728 weight_to_token_ratio=0.281122 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220039577e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130990 payload_rows_remote=114680 dup_rows=131154 dup_rows_remote=116366 dedup_ratio=0.500313 router_tokens_per_expert=[4, 0, 32, 4, 98, 3, 0, 1, 7402, 6, 8188, 8183, 2, 8132, 4, 3, 76, 0, 4, 0, 8181, 0, 475, 0, 0, 8156, 8187, 3, 4, 8175, 4, 0, 3, 0, 8179, 6, 4, 0, 0, 0, 4, 8188, 0, 1, 1, 3, 7785, 4, 8187, 8183, 3, 3, 0, 7417, 0, 3, 0, 3, 4, 0, 1390, 8188, 0, 8186] payload_rows_per_dst=[16384, 16310, 16382, 16380, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16642, 14788, 17464, 16304, 16374, 15588, 14836, 19158] combine_rows_per_src=[16384, 16310, 16382, 16380, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131026 payload_rows_remote=114642 dup_rows=131118 dup_rows_remote=114724 dedup_ratio=0.500175 router_tokens_per_expert=[0, 0, 8192, 8179, 5, 0, 13, 0, 0, 0, 0, 0, 2452, 0, 6853, 1948, 8192, 0, 0, 0, 1, 0, 2, 6908, 8192, 0, 658, 0, 0, 0, 0, 8152, 2, 0, 8171, 0, 6237, 8192, 3633, 8190, 0, 0, 7811, 456, 4157, 0, 8192, 0, 0, 0, 0, 3171, 0, 259, 0, 0, 4518, 0, 8155, 63, 1, 83, 0, 34] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16342, 16380, 16384] dup_rows_per_dst=[16394, 22464, 29702, 17620, 19744, 9010, 6860, 9324] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16342, 16380, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=938655744 dispatch_meta_bytes=906120 combine_payload_bytes=938655744 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=939561864 ingress_bytes=1643298816 link_bytes=1643298816 token_bytes=1878217608 weight_movement_bytes=704643072 total_nvlink_bytes=2582860680 weight_to_token_ratio=0.375166 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.022727 gemm_active_groups=13 gemm_peak_group_rows=30080 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.201787423e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=914280 combine_payload_bytes=939311104 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940225384 ingress_bytes=1115471872 link_bytes=1115471872 token_bytes=1879536488 weight_movement_bytes=176160768 total_nvlink_bytes=2055697256 weight_to_token_ratio=0.093726 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246176862e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939245568 dispatch_meta_bytes=921672 combine_payload_bytes=939245568 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940167240 ingress_bytes=1291567104 link_bytes=1291567104 token_bytes=1879412808 weight_movement_bytes=352321536 total_nvlink_bytes=2231734344 weight_to_token_ratio=0.187464 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232236656e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114736 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 0, 0, 10, 0, 1, 0, 7194, 0, 8192, 8183, 0, 8171, 15, 0, 45, 0, 0, 12, 8190, 0, 341, 0, 0, 8173, 8192, 0, 0, 8192, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 7698, 0, 8192, 8192, 0, 0, 0, 7824, 0, 0, 0, 0, 0, 1, 1503, 8192, 0, 8192] payload_rows_per_dst=[16384, 16372, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16388, 14388, 17176, 16346, 16348, 15398, 15648, 19392] combine_rows_per_src=[16384, 16372, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=924184 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940382744 ingress_bytes=1467940864 link_bytes=1467940864 token_bytes=1879841304 weight_movement_bytes=528482304 total_nvlink_bytes=2408323608 weight_to_token_ratio=0.281131 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219856090e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 8, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=917464 combine_payload_bytes=939147264 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940064728 ingress_bytes=1115308032 link_bytes=1115308032 token_bytes=1879211992 weight_movement_bytes=176160768 total_nvlink_bytes=2055372760 weight_to_token_ratio=0.093742 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245259427e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=917648 combine_payload_bytes=939425792 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940343440 ingress_bytes=1291747328 link_bytes=1291747328 token_bytes=1879769232 weight_movement_bytes=352321536 total_nvlink_bytes=2232090768 weight_to_token_ratio=0.187428 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233245835e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114690 dedup_ratio=0.500031 router_tokens_per_expert=[0, 0, 0, 8192, 8178, 0, 0, 8192, 0, 0, 0, 0, 8181, 22, 0, 8191, 8192, 0, 0, 8189, 0, 0, 7727, 8190, 29, 0, 216, 13, 0, 8192, 0, 0, 0, 1167, 8192, 1, 0, 0, 0, 3, 3, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 8192, 0, 7255, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[16356, 16404, 15454, 16896, 11478, 16390, 16384, 21718] combine_rows_per_src=[16384, 16384, 16378, 16384, 16384, 16384, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=114108 dedup_ratio=0.500046 router_tokens_per_expert=[0, 0, 0, 8192, 8160, 0, 0, 8192, 0, 1, 0, 0, 8179, 21, 0, 8192, 8192, 0, 0, 8186, 0, 0, 7674, 8189, 87, 2, 197, 13, 0, 8192, 0, 0, 0, 2356, 8192, 4, 0, 0, 0, 4, 3, 0, 8190, 0, 0, 0, 4, 0, 8187, 0, 0, 0, 1, 0, 0, 0, 0, 8119, 8192, 0, 6151, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16372, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16320, 16402, 15348, 16976, 21112, 16394, 16376, 12156] combine_rows_per_src=[16384, 16384, 16372, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=103698 dedup_ratio=0.500038 router_tokens_per_expert=[0, 0, 0, 8191, 8136, 1, 0, 8191, 1, 0, 1, 1, 8174, 74, 1, 8161, 8191, 0, 1, 8186, 1, 0, 7620, 8189, 91, 0, 907, 45, 1, 8191, 2, 1, 2, 2304, 8191, 0, 1, 0, 0, 1, 0, 0, 8191, 0, 1, 0, 0, 1, 8191, 0, 1, 0, 0, 1, 0, 1, 0, 8137, 8191, 0, 5499, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16272, 16330, 15186, 18470, 4614, 16384, 16442, 27384] combine_rows_per_src=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131038 payload_rows_remote=114656 dup_rows=131106 dup_rows_remote=118478 dedup_ratio=0.500130 router_tokens_per_expert=[13, 0, 0, 8182, 7447, 9, 0, 8181, 3, 32, 10, 9, 8140, 20, 9, 8010, 8181, 0, 10, 8182, 6, 0, 6307, 8181, 13, 0, 418, 4, 9, 8169, 52, 0, 181, 4359, 8175, 2, 10, 0, 0, 1, 1, 10, 8178, 0, 9, 7, 8, 0, 8150, 0, 11, 0, 0, 9, 0, 8, 0, 8154, 8183, 0, 6001, 13, 0, 5] payload_rows_per_dst=[16384, 16356, 16382, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[14916, 16110, 12628, 17308, 25438, 16406, 16334, 11966] combine_rows_per_src=[16384, 16356, 16382, 16384, 16384, 16384, 16384, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917480 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940376040 ingress_bytes=1115619328 link_bytes=1115619328 token_bytes=1879834600 weight_movement_bytes=176160768 total_nvlink_bytes=2055995368 weight_to_token_ratio=0.093711 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247002554e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114694 dedup_ratio=0.500015 router_tokens_per_expert=[0, 1, 0, 8191, 8052, 1, 0, 8191, 0, 2, 1, 1, 8188, 0, 1, 8178, 8191, 0, 0, 8191, 1, 0, 7679, 8191, 0, 0, 168, 1, 1, 8191, 16, 0, 30, 2042, 8191, 0, 1, 3, 0, 0, 0, 0, 8191, 0, 1, 1, 0, 0, 8188, 1, 1, 0, 0, 1, 0, 1, 0, 8165, 8191, 1, 6625, 0, 0, 1] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16106, 16362, 15358, 16752, 4150, 16384, 16382, 29582] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 2, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=915136 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940340928 ingress_bytes=1115586560 link_bytes=1115586560 token_bytes=1879766720 weight_movement_bytes=176160768 total_nvlink_bytes=2055927488 weight_to_token_ratio=0.093714 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.021780 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246819067e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131010 payload_rows_remote=114656 dup_rows=131134 dup_rows_remote=114886 dedup_ratio=0.500237 router_tokens_per_expert=[0, 0, 1, 8187, 7798, 4, 0, 8187, 2, 1, 4, 4, 8167, 7, 4, 8112, 8188, 0, 2, 8186, 3, 0, 6688, 8186, 13, 0, 319, 10, 4, 8188, 29, 0, 166, 4115, 8188, 0, 4, 0, 0, 0, 0, 3, 8186, 0, 4, 3, 0, 3, 8184, 3, 4, 0, 0, 4, 0, 3, 0, 8059, 8188, 1, 5654, 3, 0, 3] payload_rows_per_dst=[16384, 16354, 16380, 16384, 16384, 16384, 16384, 16356] dup_rows_per_dst=[15596, 16248, 13378, 17114, 24938, 16388, 16388, 11084] combine_rows_per_src=[16384, 16354, 16380, 16384, 16384, 16384, 16384, 16356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=123062 payload_rows_remote=106678 dup_rows=139082 dup_rows_remote=122710 dedup_ratio=0.530556 router_tokens_per_expert=[0, 0, 0, 0, 10, 0, 2, 0, 7244, 0, 8192, 8174, 0, 8166, 24, 0, 71, 0, 0, 24, 8190, 0, 190, 0, 0, 8176, 8192, 0, 0, 8189, 0, 0, 0, 0, 7905, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 6713, 0, 8192, 8192, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 2724, 8192, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 8374] dup_rows_per_dst=[16372, 27400, 16950, 17892, 15810, 24444, 16974, 3240] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 8374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=109164 dedup_ratio=0.500008 router_tokens_per_expert=[0, 0, 0, 8192, 8172, 0, 0, 8192, 0, 3, 0, 0, 8182, 60, 0, 8192, 8192, 0, 0, 8191, 0, 0, 7446, 8191, 55, 1, 227, 21, 0, 8192, 0, 0, 0, 2762, 8192, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 8192, 0, 5869, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16344, 16490, 14892, 16990, 21910, 16384, 16384, 11680] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=873504 combine_payload_bytes=939442176 prefetch_bytes=88080384 reduce_bytes=616562688 egress_bytes=940315680 ingress_bytes=1644085248 link_bytes=1644085248 token_bytes=1879757856 weight_movement_bytes=704643072 total_nvlink_bytes=2584400928 weight_to_token_ratio=0.374858 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=30592 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206191114e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=932536 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=940194488 ingress_bytes=1643905024 link_bytes=1643905024 token_bytes=1879456440 weight_movement_bytes=704643072 total_nvlink_bytes=2584099512 weight_to_token_ratio=0.374919 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.022727 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.205181935e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 8, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917512 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940408840 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879900168 weight_movement_bytes=352321536 total_nvlink_bytes=2232221704 weight_to_token_ratio=0.187415 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=918168 combine_payload_bytes=939261952 prefetch_bytes=88080384 reduce_bytes=88080384 egress_bytes=940180120 ingress_bytes=1115422720 link_bytes=1115422720 token_bytes=1879442072 weight_movement_bytes=176160768 total_nvlink_bytes=2055602840 weight_to_token_ratio=0.093730 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245901632e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=2 dispatch_payload_bytes=873906176 dispatch_meta_bytes=917552 combine_payload_bytes=873906176 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=874823728 ingress_bytes=1226227712 link_bytes=1226227712 token_bytes=1748729904 weight_movement_bytes=352321536 total_nvlink_bytes=2101051440 weight_to_token_ratio=0.201473 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.018939 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.866363335e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=895400 combine_payload_bytes=939507712 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940403112 ingress_bytes=1115668480 link_bytes=1115668480 token_bytes=1879910824 weight_movement_bytes=176160768 total_nvlink_bytes=2056071592 weight_to_token_ratio=0.093707 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.021780 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247277785e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130998 payload_rows_remote=114646 dup_rows=131146 dup_rows_remote=105510 dedup_ratio=0.500282 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 13, 0, 0, 2154, 384, 158, 0, 0, 0, 0, 8192, 0, 8192, 0, 8181, 0, 100, 0, 0, 8152, 0, 0, 0, 6228, 8160, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 2571, 8133, 2491, 0, 8192, 8188, 0, 0, 0, 5, 0, 0, 7087, 0, 8189, 0, 0, 2085, 1405, 0, 44, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16342, 16384, 16352, 16384, 16384] dup_rows_per_dst=[16388, 7004, 16584, 11600, 16384, 25636, 14178, 23372] combine_rows_per_src=[16384, 16384, 16384, 16342, 16384, 16352, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130796 payload_rows_remote=114440 dup_rows=131348 dup_rows_remote=101864 dedup_ratio=0.501053 router_tokens_per_expert=[2, 0, 0, 8192, 0, 0, 26, 0, 0, 1828, 566, 55, 0, 0, 0, 0, 8192, 0, 8192, 0, 8180, 0, 309, 0, 0, 8164, 0, 0, 0, 6698, 8058, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 2208, 8014, 2535, 0, 8192, 8192, 0, 0, 0, 9, 0, 0, 6766, 0, 8186, 0, 0, 2345, 1494, 0, 93, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16356, 16384, 16136, 16384, 16384] dup_rows_per_dst=[16416, 21282, 17002, 29484, 16384, 9378, 13538, 7864] combine_rows_per_src=[16384, 16384, 16384, 16356, 16384, 16136, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=117628 payload_rows_remote=101244 dup_rows=144516 dup_rows_remote=114672 dedup_ratio=0.551285 router_tokens_per_expert=[0, 0, 0, 8190, 0, 0, 8, 18, 0, 1713, 274, 96, 0, 1, 1, 0, 8189, 1, 8182, 1, 8186, 1, 284, 1, 0, 8129, 1, 1, 1, 5623, 7894, 1, 0, 8191, 0, 0, 8188, 0, 0, 1, 11, 2963, 7959, 760, 0, 8191, 8191, 0, 0, 1, 2, 3, 1, 6641, 0, 8188, 1, 1, 4704, 2004, 0, 83, 1, 8191] payload_rows_per_dst=[16382, 16384, 16384, 2944, 16384, 16382, 16384, 16384] dup_rows_per_dst=[15502, 4168, 14870, 6, 16418, 34632, 29076, 29844] combine_rows_per_src=[16382, 16384, 16384, 2944, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939180032 dispatch_meta_bytes=880624 combine_payload_bytes=939180032 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940060656 ingress_bytes=1467662336 link_bytes=1467662336 token_bytes=1879240688 weight_movement_bytes=528482304 total_nvlink_bytes=2407722992 weight_to_token_ratio=0.281221 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.024621 gemm_active_groups=14 gemm_peak_group_rows=32384 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218296449e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130676 payload_rows_remote=114294 dup_rows=131468 dup_rows_remote=115022 dedup_ratio=0.501511 router_tokens_per_expert=[0, 0, 0, 8175, 0, 0, 19, 299, 0, 2541, 262, 58, 0, 10, 10, 0, 7777, 10, 7606, 15, 8158, 11, 988, 7, 0, 8163, 9, 9, 8, 6226, 6820, 604, 4, 8181, 1, 0, 8170, 0, 10, 11, 48, 1320, 7979, 618, 0, 8169, 8136, 0, 0, 10, 2, 15, 8, 6115, 0, 8180, 32, 9, 6426, 1651, 0, 13, 0, 8179] payload_rows_per_dst=[16368, 16358, 16382, 16382, 16384, 16034, 16384, 16384] dup_rows_per_dst=[16934, 22014, 16446, 27296, 16370, 3896, 12276, 16236] combine_rows_per_src=[16368, 16358, 16382, 16382, 16384, 16034, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937492480 dispatch_meta_bytes=865216 combine_payload_bytes=937492480 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=938357696 ingress_bytes=1642135552 link_bytes=1642135552 token_bytes=1875850176 weight_movement_bytes=704643072 total_nvlink_bytes=2580493248 weight_to_token_ratio=0.375639 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=25216 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.195273630e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=122708 payload_rows_remote=106324 dup_rows=139436 dup_rows_remote=117530 dedup_ratio=0.531906 router_tokens_per_expert=[0, 0, 0, 8191, 1, 0, 0, 51, 0, 3336, 73, 65, 0, 1, 1, 0, 8099, 1, 8046, 2, 8179, 1, 176, 1, 0, 8186, 1, 1, 1, 6892, 7676, 127, 1, 8191, 1, 0, 8184, 0, 0, 1, 4, 1285, 8186, 855, 0, 8191, 8187, 0, 4, 1, 1, 4, 1, 6814, 0, 8191, 1, 1, 4916, 745, 0, 9, 0, 8190] payload_rows_per_dst=[16384, 16384, 16376, 8030, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16460, 6950, 16276, 2430, 16372, 31330, 21906, 27712] combine_rows_per_src=[16384, 16384, 16376, 8030, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131044 payload_rows_remote=114670 dup_rows=131100 dup_rows_remote=109366 dedup_ratio=0.500107 router_tokens_per_expert=[0, 0, 0, 8187, 2, 0, 0, 106, 0, 2503, 27, 166, 0, 4, 4, 2, 7846, 4, 8013, 5, 8184, 4, 568, 4, 0, 8172, 4, 4, 4, 7484, 7109, 237, 3, 8188, 1, 0, 8181, 2, 2, 4, 26, 2525, 8143, 528, 0, 8182, 8166, 0, 0, 4, 0, 10, 3, 6728, 0, 8188, 7, 4, 3971, 1375, 0, 0, 0, 8188] payload_rows_per_dst=[16380, 16374, 16382, 16384, 16384, 16372, 16384, 16384] dup_rows_per_dst=[16578, 21734, 16506, 29644, 16378, 6072, 13482, 10706] combine_rows_per_src=[16380, 16374, 16382, 16384, 16384, 16372, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130138 payload_rows_remote=113754 dup_rows=132006 dup_rows_remote=112844 dedup_ratio=0.503563 router_tokens_per_expert=[0, 0, 0, 8192, 8180, 0, 0, 8192, 0, 16, 0, 0, 8185, 50, 0, 8185, 8192, 0, 0, 8188, 0, 0, 8000, 8191, 57, 1, 584, 19, 0, 8192, 0, 0, 0, 522, 8192, 1, 0, 0, 0, 1, 3, 0, 8191, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8192, 0, 7173, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 15452] dup_rows_per_dst=[19162, 16788, 28866, 10566, 17432, 16342, 16382, 6468] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 15452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130788 payload_rows_remote=114404 dup_rows=131356 dup_rows_remote=114972 dedup_ratio=0.501083 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 16, 0, 0, 3344, 581, 50, 0, 0, 0, 0, 8192, 0, 8192, 0, 8189, 0, 109, 0, 0, 8151, 0, 0, 0, 6594, 8009, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 2759, 8016, 1314, 0, 8192, 8192, 0, 0, 0, 2, 0, 0, 6711, 0, 8191, 0, 0, 2118, 1289, 0, 93, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16104, 16384, 16384] dup_rows_per_dst=[16410, 22220, 16602, 24096, 16384, 10188, 13424, 12032] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16104, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=829390848 dispatch_meta_bytes=863664 combine_payload_bytes=829390848 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=830254512 ingress_bytes=1005551616 link_bytes=1005551616 token_bytes=1659645360 weight_movement_bytes=176160768 total_nvlink_bytes=1835806128 weight_to_token_ratio=0.106144 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.022727 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.630669312e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=936296448 dispatch_meta_bytes=917264 combine_payload_bytes=936296448 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=937213712 ingress_bytes=1464778752 link_bytes=1464778752 token_bytes=1873510160 weight_movement_bytes=528482304 total_nvlink_bytes=2401992464 weight_to_token_ratio=0.282081 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.202149582e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=871006208 dispatch_meta_bytes=895416 combine_payload_bytes=871006208 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=871901624 ingress_bytes=1047166976 link_bytes=1047166976 token_bytes=1742907832 weight_movement_bytes=176160768 total_nvlink_bytes=1919068600 weight_to_token_ratio=0.101073 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=5.863697957e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=896144 combine_payload_bytes=939376640 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940272784 ingress_bytes=1291698176 link_bytes=1291698176 token_bytes=1879649424 weight_movement_bytes=352321536 total_nvlink_bytes=2231970960 weight_to_token_ratio=0.187440 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232970604e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=2 dispatch_payload_bytes=931872768 dispatch_meta_bytes=906392 combine_payload_bytes=931872768 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=932779160 ingress_bytes=1284194304 link_bytes=1284194304 token_bytes=1864651928 weight_movement_bytes=352321536 total_nvlink_bytes=2216973464 weight_to_token_ratio=0.188948 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.190952053e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114376 dedup_ratio=0.500015 router_tokens_per_expert=[8192, 0, 8184, 8192, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 8145, 2564, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 4, 8192, 0, 43, 0, 7072, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 5, 156, 0, 0, 8192, 0, 0, 8192, 0, 6588, 0, 0, 8191, 9, 8192, 1, 0, 8192] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16368, 16290, 5128, 16392, 14236, 16700, 29560, 16402] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937197568 dispatch_meta_bytes=917504 combine_payload_bytes=937197568 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938115072 ingress_bytes=1113358336 link_bytes=1113358336 token_bytes=1875312640 weight_movement_bytes=176160768 total_nvlink_bytes=2051473408 weight_to_token_ratio=0.093937 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.234341943e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114676 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 8181, 8192, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 8123, 3814, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 6, 8192, 0, 246, 0, 6975, 0, 0, 0, 9, 0, 0, 0, 0, 8191, 0, 0, 2, 164, 0, 0, 8192, 0, 0, 8192, 0, 5237, 0, 0, 8191, 15, 8192, 3, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16362, 16246, 20956, 16396, 14460, 16706, 13528, 16418] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114656 dup_rows=131104 dup_rows_remote=114620 dedup_ratio=0.500122 router_tokens_per_expert=[8191, 0, 8184, 8188, 2, 1, 20, 1, 56, 8189, 8176, 1, 0, 0, 1, 8137, 4633, 0, 1, 1, 1, 0, 1, 8191, 0, 1, 1, 0, 1, 1, 8190, 0, 85, 0, 7588, 0, 0, 0, 21, 0, 0, 0, 1, 8191, 0, 1, 59, 142, 0, 3, 8190, 0, 2, 8191, 1, 3846, 0, 1, 8189, 10, 8191, 0, 0, 8191] payload_rows_per_dst=[16384, 16354, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[6818, 16248, 9272, 16370, 24978, 16782, 24152, 16484] combine_rows_per_src=[16384, 16354, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=916240 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940407568 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879898896 weight_movement_bytes=176160768 total_nvlink_bytes=2056059664 weight_to_token_ratio=0.093708 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.024621 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130998 payload_rows_remote=114616 dup_rows=131146 dup_rows_remote=107070 dedup_ratio=0.500282 router_tokens_per_expert=[8181, 0, 8180, 8149, 28, 10, 91, 9, 291, 8181, 8006, 10, 0, 15, 8, 7832, 3843, 1, 10, 11, 9, 0, 9, 8180, 6, 9, 10, 0, 7, 2, 8120, 0, 1155, 7, 7769, 44, 9, 0, 12, 0, 0, 0, 10, 8170, 0, 10, 28, 1231, 0, 48, 8153, 112, 0, 8166, 6, 2389, 0, 6, 8177, 2, 8181, 1, 0, 8178] payload_rows_per_dst=[16384, 16330, 16382, 16384, 16384, 16384, 16382, 16368] dup_rows_per_dst=[16550, 15994, 24076, 16286, 17970, 18876, 5034, 16360] combine_rows_per_src=[16384, 16330, 16382, 16384, 16384, 16384, 16382, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917456 combine_payload_bytes=939524096 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940441552 ingress_bytes=1203765248 link_bytes=1203765248 token_bytes=1879965648 weight_movement_bytes=264241152 total_nvlink_bytes=2144206800 weight_to_token_ratio=0.140556 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740582912e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 3, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131052 payload_rows_remote=114670 dup_rows=131092 dup_rows_remote=109492 dedup_ratio=0.500076 router_tokens_per_expert=[8191, 0, 8191, 8186, 5, 0, 13, 1, 80, 8191, 8173, 1, 0, 2, 0, 8087, 5156, 1, 1, 1, 1, 1, 1, 8189, 0, 1, 1, 0, 1, 0, 8188, 0, 360, 1, 7913, 3, 1, 0, 1, 0, 0, 0, 1, 8190, 0, 1, 19, 350, 0, 4, 8190, 12, 0, 8190, 0, 2595, 0, 1, 8191, 0, 8191, 0, 4, 8191] payload_rows_per_dst=[16384, 16368, 16382, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16408, 16318, 10320, 16380, 16556, 17120, 21600, 16390] combine_rows_per_src=[16384, 16368, 16382, 16384, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131032 payload_rows_remote=114680 dup_rows=131112 dup_rows_remote=114514 dedup_ratio=0.500153 router_tokens_per_expert=[8188, 0, 8188, 8176, 24, 2, 45, 4, 269, 8188, 8099, 4, 0, 0, 5, 8098, 6249, 3, 4, 5, 4, 2, 4, 8188, 0, 4, 4, 0, 3, 0, 8174, 0, 102, 3, 8067, 5, 4, 10, 4, 0, 0, 0, 4, 8186, 0, 4, 70, 609, 7, 13, 8182, 14, 1, 8188, 0, 1096, 0, 3, 8183, 0, 8188, 0, 7, 8188] payload_rows_per_dst=[16384, 16352, 16384, 16384, 16384, 16384, 16378, 16382] dup_rows_per_dst=[16494, 16598, 28910, 16362, 16382, 17738, 2248, 16380] combine_rows_per_src=[16384, 16352, 16384, 16384, 16384, 16384, 16378, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=917104 combine_payload_bytes=939261952 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940179056 ingress_bytes=1379663872 link_bytes=1379663872 token_bytes=1879441008 weight_movement_bytes=440401920 total_nvlink_bytes=2319842928 weight_to_token_ratio=0.234326 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.725541783e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130856 payload_rows_remote=114472 dup_rows=131288 dup_rows_remote=114882 dedup_ratio=0.500824 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 25, 0, 0, 4003, 604, 72, 0, 0, 0, 0, 8192, 0, 8190, 0, 8178, 0, 209, 0, 0, 8148, 0, 0, 0, 6160, 8077, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 1952, 8044, 3629, 0, 8191, 8190, 0, 0, 0, 3, 0, 0, 6594, 0, 8191, 0, 0, 754, 838, 0, 61, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16372, 16384, 16180, 16384, 16384] dup_rows_per_dst=[16406, 25736, 16798, 28398, 16382, 11070, 13192, 3306] combine_rows_per_src=[16384, 16384, 16384, 16372, 16384, 16180, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114686 dup_rows=131074 dup_rows_remote=116170 dedup_ratio=0.500008 router_tokens_per_expert=[8192, 0, 8187, 8192, 0, 0, 0, 0, 11, 8191, 8192, 0, 0, 0, 0, 8182, 2666, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 8192, 0, 323, 0, 7124, 0, 0, 0, 5, 0, 0, 0, 0, 8191, 0, 0, 1, 406, 0, 0, 8192, 0, 0, 8192, 0, 5854, 0, 0, 8192, 7, 8192, 1, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16374, 16386, 8198, 16390, 14904, 17196, 25226, 16400] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=938934272 dispatch_meta_bytes=886744 combine_payload_bytes=938934272 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939821016 ingress_bytes=1115095040 link_bytes=1115095040 token_bytes=1878755288 weight_movement_bytes=176160768 total_nvlink_bytes=2054916056 weight_to_token_ratio=0.093765 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244066760e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939376640 dispatch_meta_bytes=896648 combine_payload_bytes=939376640 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940273288 ingress_bytes=1467858944 link_bytes=1467858944 token_bytes=1879649928 weight_movement_bytes=528482304 total_nvlink_bytes=2408132232 weight_to_token_ratio=0.281160 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219397372e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=916776 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940375336 ingress_bytes=1467940864 link_bytes=1467940864 token_bytes=1879833896 weight_movement_bytes=528482304 total_nvlink_bytes=2408316200 weight_to_token_ratio=0.281132 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.019886 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219856090e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114688 dup_rows=131082 dup_rows_remote=115836 dedup_ratio=0.500038 router_tokens_per_expert=[8192, 0, 51, 0, 0, 0, 8189, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 263, 0, 0, 0, 0, 3, 0, 8165, 8192, 0, 0, 8192, 0, 8192, 8192, 8192, 1361, 0, 0, 0, 0, 0, 8185, 6750, 0, 0, 1, 0, 7974, 8192, 8192, 0, 0, 0, 21, 0, 0, 8190, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16384] dup_rows_per_dst=[16472, 16384, 16902, 16336, 16936, 15246, 16426, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=2 dispatch_payload_bytes=937754624 dispatch_meta_bytes=917416 combine_payload_bytes=937754624 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938672040 ingress_bytes=1113915392 link_bytes=1113915392 token_bytes=1876426664 weight_movement_bytes=176160768 total_nvlink_bytes=2052587432 weight_to_token_ratio=0.093881 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.025568 gemm_active_groups=13 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.237461224e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=939507712 dispatch_meta_bytes=923424 combine_payload_bytes=939507712 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940431136 ingress_bytes=1291829248 link_bytes=1291829248 token_bytes=1879938848 weight_movement_bytes=352321536 total_nvlink_bytes=2232260384 weight_to_token_ratio=0.187411 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233704553e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130912 payload_rows_remote=114528 dup_rows=131232 dup_rows_remote=114838 dedup_ratio=0.500610 router_tokens_per_expert=[8192, 0, 12, 0, 0, 0, 8187, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 229, 0, 0, 0, 0, 8, 0, 8189, 8192, 0, 0, 8192, 0, 8192, 8192, 8192, 2278, 0, 2, 0, 0, 0, 8157, 6407, 0, 0, 3, 3, 7432, 8192, 8192, 0, 0, 0, 62, 0, 0, 8189, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16378, 16384, 16384, 16230, 16384, 16384] dup_rows_per_dst=[16362, 16384, 16826, 16394, 20920, 11460, 16508, 16378] combine_rows_per_src=[16384, 16384, 16378, 16384, 16384, 16230, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=922096 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940446192 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879970288 weight_movement_bytes=352321536 total_nvlink_bytes=2232291824 weight_to_token_ratio=0.187408 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=28928 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=129134 payload_rows_remote=112762 dup_rows=133010 dup_rows_remote=114378 dedup_ratio=0.507393 router_tokens_per_expert=[8158, 0, 68, 1, 1, 0, 8191, 8189, 9, 1, 1, 0, 0, 0, 1, 1, 2, 35, 0, 1118, 1, 12, 1, 73, 0, 1, 8168, 8191, 0, 0, 8191, 5, 8191, 8191, 8185, 522, 0, 0, 0, 1, 1, 8181, 7250, 5, 0, 25, 0, 7390, 8190, 8191, 0, 5, 0, 5, 0, 0, 8132, 1, 1, 8185, 0, 1, 0, 0] payload_rows_per_dst=[16318, 16384, 16384, 14538, 16372, 16384, 16382, 16372] dup_rows_per_dst=[138, 15588, 18860, 8074, 10054, 45264, 16400, 18632] combine_rows_per_src=[16318, 16384, 16384, 14538, 16372, 16384, 16382, 16372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938213376 dispatch_meta_bytes=917464 combine_payload_bytes=938213376 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=939130840 ingress_bytes=1642856448 link_bytes=1642856448 token_bytes=1877344216 weight_movement_bytes=704643072 total_nvlink_bytes=2581987288 weight_to_token_ratio=0.375340 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.199310347e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130540 payload_rows_remote=114158 dup_rows=131604 dup_rows_remote=107720 dedup_ratio=0.502029 router_tokens_per_expert=[7687, 0, 323, 9, 9, 0, 8180, 8182, 117, 12, 7, 0, 0, 1, 10, 9, 0, 5, 0, 2901, 10, 234, 8, 613, 1, 7, 8050, 8176, 0, 0, 8178, 37, 8180, 8182, 7936, 1798, 0, 0, 2, 11, 12, 8141, 5822, 74, 0, 13, 0, 5746, 8171, 8180, 0, 311, 0, 5, 0, 0, 7656, 11, 11, 8021, 13, 10, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16380, 16378, 16194, 16368, 16070] dup_rows_per_dst=[15962, 16652, 23884, 16162, 19464, 7140, 16966, 15374] combine_rows_per_src=[16384, 16384, 16382, 16380, 16378, 16194, 16368, 16070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130558 payload_rows_remote=114174 dup_rows=131586 dup_rows_remote=114828 dedup_ratio=0.501961 router_tokens_per_expert=[7955, 0, 173, 3, 4, 0, 8188, 8189, 181, 4, 3, 0, 0, 0, 4, 4, 0, 4, 0, 3870, 4, 92, 4, 442, 0, 3, 7922, 8187, 0, 2, 8187, 16, 8187, 8188, 7943, 52, 0, 0, 2, 4, 4, 8152, 6451, 27, 0, 15, 0, 6128, 8180, 8188, 0, 69, 4, 2, 0, 0, 7909, 4, 4, 8111, 2, 4, 1, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16070, 16338, 16384, 16232] dup_rows_per_dst=[24494, 16758, 16932, 16812, 15338, 8912, 16502, 15838] combine_rows_per_src=[16384, 16384, 16384, 16382, 16070, 16338, 16384, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=923746304 dispatch_meta_bytes=908560 combine_payload_bytes=923746304 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=924654864 ingress_bytes=1099907072 link_bytes=1099907072 token_bytes=1848401168 weight_movement_bytes=176160768 total_nvlink_bytes=2024561936 weight_to_token_ratio=0.095304 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.018939 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.159020479e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[8, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130912 payload_rows_remote=114528 dup_rows=131232 dup_rows_remote=114838 dedup_ratio=0.500610 router_tokens_per_expert=[8120, 0, 205, 1, 1, 0, 8191, 8191, 69, 1, 1, 0, 1, 0, 1, 1, 0, 4, 0, 1408, 1, 26, 1, 170, 0, 1, 7853, 8190, 0, 3, 8191, 2, 8190, 8191, 8121, 1192, 0, 0, 1, 1, 1, 8170, 6472, 5, 0, 5, 0, 7418, 8190, 8191, 0, 5, 1, 2, 0, 0, 8094, 1, 1, 8186, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16250, 16368, 16384, 16374] dup_rows_per_dst=[7272, 16526, 19600, 17270, 10572, 27406, 16394, 16192] combine_rows_per_src=[16384, 16384, 16384, 16384, 16250, 16368, 16384, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130150 payload_rows_remote=113766 dup_rows=131994 dup_rows_remote=106296 dedup_ratio=0.503517 router_tokens_per_expert=[8192, 0, 8179, 8192, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 7425, 3918, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 8192, 0, 97, 0, 5695, 0, 0, 0, 2, 0, 0, 0, 0, 8191, 0, 0, 3, 226, 0, 0, 8192, 0, 0, 8192, 0, 7207, 0, 0, 8192, 15, 8192, 2, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 15464, 16384] dup_rows_per_dst=[25698, 17124, 24220, 16496, 11588, 9958, 10654, 16256] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 15464, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130990 payload_rows_remote=114606 dup_rows=131154 dup_rows_remote=110370 dedup_ratio=0.500313 router_tokens_per_expert=[8192, 0, 491, 0, 0, 0, 8191, 8188, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 1, 0, 8183, 8192, 0, 0, 8192, 0, 8191, 8192, 8192, 2205, 0, 1, 0, 0, 0, 8186, 5379, 0, 0, 1, 0, 7771, 8192, 8192, 0, 0, 0, 12, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16304, 16384, 16384] dup_rows_per_dst=[17366, 16392, 17454, 16368, 20784, 9998, 16408, 16384] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16304, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=935182336 dispatch_meta_bytes=887512 combine_payload_bytes=935182336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936069848 ingress_bytes=1287503872 link_bytes=1287503872 token_bytes=1871252184 weight_movement_bytes=352321536 total_nvlink_bytes=2223573720 weight_to_token_ratio=0.188281 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.209484253e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=935313408 dispatch_meta_bytes=916008 combine_payload_bytes=935313408 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=936229416 ingress_bytes=1287634944 link_bytes=1287634944 token_bytes=1871542824 weight_movement_bytes=352321536 total_nvlink_bytes=2223864360 weight_to_token_ratio=0.188252 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.017992 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.210218201e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=115104 dedup_ratio=0.500000 router_tokens_per_expert=[8166, 8192, 1, 0, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 8188, 0, 8089, 8192, 8192, 0, 0, 0, 290, 0, 0, 8192, 16, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 4, 8189, 8192, 0, 7980, 0, 8116, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 8191, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16344, 16384, 16758, 16416, 16378, 15968, 16232, 16592] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=2 dispatch_payload_bytes=931971072 dispatch_meta_bytes=880248 combine_payload_bytes=931971072 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=932851320 ingress_bytes=1460453376 link_bytes=1460453376 token_bytes=1864822392 weight_movement_bytes=528482304 total_nvlink_bytes=2393304696 weight_to_token_ratio=0.283396 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.177929282e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938213376 dispatch_meta_bytes=917464 combine_payload_bytes=938213376 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=939130840 ingress_bytes=1114374144 link_bytes=1114374144 token_bytes=1877344216 weight_movement_bytes=176160768 total_nvlink_bytes=2053504984 weight_to_token_ratio=0.093835 gemm_rows_defined=131712 gemm_rows_charged=135168 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.025568 gemm_active_groups=12 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.240030044e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=938852352 dispatch_meta_bytes=899904 combine_payload_bytes=938852352 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=939752256 ingress_bytes=1467334656 link_bytes=1467334656 token_bytes=1878604608 weight_movement_bytes=528482304 total_nvlink_bytes=2407086912 weight_to_token_ratio=0.281316 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.216461578e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114622 dedup_ratio=0.500000 router_tokens_per_expert=[8143, 8192, 0, 0, 0, 18, 0, 0, 6, 0, 0, 0, 0, 0, 8186, 0, 8115, 8192, 8192, 0, 0, 0, 414, 2, 0, 8192, 33, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 8192, 6, 8188, 8192, 0, 7743, 0, 8153, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 8188, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[16322, 16384, 17062, 16450, 16380, 15498, 16306, 16670] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 5, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128524 payload_rows_remote=112140 dup_rows=133620 dup_rows_remote=116154 dedup_ratio=0.509720 router_tokens_per_expert=[7163, 8189, 0, 310, 1, 3, 0, 0, 0, 4, 0, 1, 0, 0, 8186, 66, 7574, 8101, 8191, 0, 0, 0, 1950, 0, 1, 8191, 7, 0, 1, 3, 8190, 8191, 0, 1, 0, 1, 1, 0, 0, 0, 1, 8191, 8187, 4, 8191, 8160, 1, 7574, 46, 7984, 1, 2, 0, 0, 2, 0, 212, 0, 0, 1, 8184, 0, 4, 1] payload_rows_per_dst=[16384, 16384, 13898, 16384, 16384, 16322, 16384, 16384] dup_rows_per_dst=[14948, 14922, 11922, 16274, 16356, 21318, 20414, 17466] combine_rows_per_src=[16384, 16384, 13898, 16384, 16384, 16322, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 3] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=919168 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=940443264 ingress_bytes=1732247552 link_bytes=1732247552 token_bytes=1879967360 weight_movement_bytes=792723456 total_nvlink_bytes=2672690816 weight_to_token_ratio=0.421669 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.699863215e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 6, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917240 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=352321536 egress_bytes=940441336 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965432 weight_movement_bytes=440401920 total_nvlink_bytes=2320367352 weight_to_token_ratio=0.234261 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114658 dup_rows=131116 dup_rows_remote=109190 dedup_ratio=0.500168 router_tokens_per_expert=[5206, 8117, 0, 1344, 9, 10, 6, 1, 1, 35, 0, 0, 1, 0, 8184, 400, 7666, 7551, 8158, 0, 0, 0, 3931, 0, 11, 8180, 27, 0, 10, 29, 8171, 8179, 0, 11, 0, 17, 8, 0, 0, 0, 8, 8182, 8125, 2, 8177, 7992, 10, 8029, 248, 7453, 0, 14, 0, 6, 4, 0, 19, 5, 60, 13, 7408, 0, 46, 8] payload_rows_per_dst=[16384, 16374, 16370, 16384, 16384, 16364, 16384, 16384] dup_rows_per_dst=[13002, 17118, 21926, 16470, 16406, 15718, 15382, 15094] combine_rows_per_src=[16384, 16374, 16370, 16384, 16384, 16364, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=918650880 dispatch_meta_bytes=913176 combine_payload_bytes=918650880 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=919564056 ingress_bytes=1182892032 link_bytes=1182892032 token_bytes=1838214936 weight_movement_bytes=264241152 total_nvlink_bytes=2102456088 weight_to_token_ratio=0.143749 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.623701616e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131016 payload_rows_remote=114638 dup_rows=131128 dup_rows_remote=114138 dedup_ratio=0.500214 router_tokens_per_expert=[5857, 8175, 0, 1118, 4, 0, 3, 2, 0, 24, 0, 1, 0, 0, 8187, 301, 7208, 7762, 8186, 0, 0, 0, 3335, 0, 5, 8187, 0, 0, 4, 10, 8184, 8187, 0, 4, 0, 5, 4, 0, 0, 2, 4, 8188, 8171, 0, 8187, 8117, 4, 8086, 162, 7250, 0, 6, 2, 1, 5, 0, 3, 3, 12, 6, 8079, 0, 28, 3] payload_rows_per_dst=[16384, 16378, 16376, 16384, 16384, 16342, 16384, 16384] dup_rows_per_dst=[13934, 16990, 20234, 16396, 16396, 16080, 14840, 16258] combine_rows_per_src=[16384, 16378, 16376, 16384, 16384, 16342, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=9 export_slots=9 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 2, 4, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130534 payload_rows_remote=114150 dup_rows=131610 dup_rows_remote=98538 dedup_ratio=0.502052 router_tokens_per_expert=[8192, 0, 169, 0, 0, 0, 8189, 8184, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 3, 0, 8191, 8192, 0, 0, 8192, 0, 8192, 8192, 8192, 4642, 0, 1, 0, 0, 0, 8179, 3920, 0, 0, 4, 0, 7339, 8192, 8192, 0, 0, 0, 15, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 15848, 16384, 16384] dup_rows_per_dst=[33072, 16388, 626, 25508, 17470, 6678, 15486, 16382] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 15848, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=115504 dedup_ratio=0.500015 router_tokens_per_expert=[6925, 8187, 0, 336, 1, 0, 1, 1, 2, 0, 1, 0, 1, 0, 8191, 73, 8059, 8047, 8190, 0, 0, 0, 1779, 0, 1, 8191, 1, 0, 1, 2, 8189, 8191, 0, 1, 0, 1, 1, 0, 0, 0, 1, 8191, 8189, 0, 8191, 8151, 1, 7996, 57, 7728, 0, 1, 1, 0, 1, 0, 21, 1, 11, 2, 8151, 0, 4, 1] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[14518, 16530, 19388, 16386, 16386, 15916, 15572, 16380] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16382, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131070 payload_rows_remote=114688 dup_rows=131074 dup_rows_remote=114692 dedup_ratio=0.500008 router_tokens_per_expert=[8187, 8192, 1, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 8189, 0, 8143, 8191, 8192, 0, 0, 0, 645, 0, 0, 8192, 7, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 3, 8191, 8192, 0, 7589, 0, 8119, 0, 0, 0, 0, 0, 1, 83, 0, 0, 0, 8182, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[16386, 16380, 17574, 16398, 16382, 15184, 16240, 16530] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939278336 dispatch_meta_bytes=895392 combine_payload_bytes=939278336 prefetch_bytes=88080384 reduce_bytes=792723456 egress_bytes=940173728 ingress_bytes=1820082176 link_bytes=1820082176 token_bytes=1879452064 weight_movement_bytes=880803840 total_nvlink_bytes=2760255904 weight_to_token_ratio=0.468649 gemm_rows_defined=131584 gemm_rows_charged=135168 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.026515 gemm_active_groups=10 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019170045e-04 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 4, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939114496 dispatch_meta_bytes=915104 combine_payload_bytes=939114496 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940029600 ingress_bytes=1115275264 link_bytes=1115275264 token_bytes=1879144096 weight_movement_bytes=176160768 total_nvlink_bytes=2055304864 weight_to_token_ratio=0.093745 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245075940e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 8, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=10 unused_prefetch_slots=22 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=5 remote_demand_peak=3 slot_utilization=0.312500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 10, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131060 payload_rows_remote=114676 dup_rows=131084 dup_rows_remote=106130 dedup_ratio=0.500046 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 6047, 0, 0, 2, 0, 429, 0, 17, 0, 0, 0, 1, 0, 0, 1, 0, 6650, 0, 11, 8192, 0, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 7334, 8004, 0, 8192, 8190, 4287, 0, 8192, 8192, 0, 4, 8190, 8192, 5, 0, 8175, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16372, 16384, 16384, 16384] dup_rows_per_dst=[16352, 12990, 16388, 13322, 14304, 24954, 16392, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16372, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=118528 dedup_ratio=0.500053 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 0, 6724, 0, 0, 2, 0, 286, 0, 35, 0, 0, 0, 3, 0, 0, 0, 0, 6268, 0, 11, 8192, 0, 0, 0, 8188, 0, 0, 0, 8192, 0, 0, 7355, 7982, 0, 8192, 8187, 4105, 0, 8188, 8192, 0, 7, 8185, 8192, 9, 0, 8186, 0, 4, 0, 8189, 8192, 0, 2, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16370, 16384, 16384, 16384] dup_rows_per_dst=[16380, 14094, 16390, 12558, 14304, 24576, 16402, 16382] combine_rows_per_src=[16384, 16384, 16384, 16384, 16370, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=2 dispatch_payload_bytes=935116800 dispatch_meta_bytes=850752 combine_payload_bytes=935116800 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=935967552 ingress_bytes=1463599104 link_bytes=1463599104 token_bytes=1871084352 weight_movement_bytes=528482304 total_nvlink_bytes=2399566656 weight_to_token_ratio=0.282447 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.023674 gemm_active_groups=14 gemm_peak_group_rows=32640 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.195544046e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=920752 combine_payload_bytes=939491328 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940412080 ingress_bytes=1291812864 link_bytes=1291812864 token_bytes=1879903408 weight_movement_bytes=352321536 total_nvlink_bytes=2232224944 weight_to_token_ratio=0.187415 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233612809e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917520 combine_payload_bytes=939524096 prefetch_bytes=440401920 reduce_bytes=0 egress_bytes=940441616 ingress_bytes=1379926016 link_bytes=1379926016 token_bytes=1879965712 weight_movement_bytes=440401920 total_nvlink_bytes=2320367632 weight_to_token_ratio=0.234261 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.727009680e-05 prefetch_bytes_per_gemm_flop=2.466066919e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 4, 2, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131028 payload_rows_remote=114644 dup_rows=131116 dup_rows_remote=114680 dedup_ratio=0.500168 router_tokens_per_expert=[1, 0, 1, 52, 0, 0, 1, 0, 5870, 0, 1, 0, 0, 1091, 0, 29, 3, 0, 1, 48, 1, 1, 1, 1, 4751, 1, 38, 8186, 9, 0, 1, 8191, 5, 0, 0, 8187, 0, 0, 6717, 8118, 4, 8191, 8181, 6102, 0, 8137, 8188, 0, 9, 8162, 8190, 4, 1, 8186, 1, 0, 1, 8188, 8191, 0, 0, 30, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16350, 16374, 16384, 16384] dup_rows_per_dst=[15142, 13972, 15876, 9588, 13330, 29084, 17688, 16436] combine_rows_per_src=[16384, 16384, 16384, 16384, 16350, 16374, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939425792 dispatch_meta_bytes=883224 combine_payload_bytes=939425792 prefetch_bytes=176160768 reduce_bytes=704643072 egress_bytes=940309016 ingress_bytes=1820229632 link_bytes=1820229632 token_bytes=1879734808 weight_movement_bytes=880803840 total_nvlink_bytes=2760538648 weight_to_token_ratio=0.468579 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=22656 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=1.019252614e-04 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=932808 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940342216 ingress_bytes=1291730944 link_bytes=1291730944 token_bytes=1879751624 weight_movement_bytes=352321536 total_nvlink_bytes=2232073160 weight_to_token_ratio=0.187430 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233154091e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939163648 dispatch_meta_bytes=917296 combine_payload_bytes=939163648 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=940080944 ingress_bytes=1115324416 link_bytes=1115324416 token_bytes=1879244592 weight_movement_bytes=176160768 total_nvlink_bytes=2055405360 weight_to_token_ratio=0.093740 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245351170e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128328 payload_rows_remote=111944 dup_rows=133816 dup_rows_remote=117194 dedup_ratio=0.510468 router_tokens_per_expert=[10, 1, 11, 350, 0, 0, 12, 0, 4017, 0, 11, 0, 0, 2734, 0, 17, 26, 0, 9, 84, 9, 11, 0, 19, 4555, 13, 20, 8106, 22, 0, 9, 8181, 22, 0, 0, 8100, 0, 6, 7452, 8164, 46, 8181, 8155, 5800, 8, 7737, 8164, 0, 4, 8162, 8150, 3, 1, 8174, 11, 1, 12, 8146, 8177, 0, 0, 159, 0, 10] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 13646, 16384, 16384] dup_rows_per_dst=[17032, 13374, 16622, 17114, 25726, 10982, 16342, 16624] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 13646, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127328 payload_rows_remote=110944 dup_rows=134816 dup_rows_remote=110072 dedup_ratio=0.514282 router_tokens_per_expert=[4, 0, 4, 270, 0, 0, 5, 0, 4485, 0, 4, 0, 0, 1431, 0, 0, 14, 0, 4, 87, 4, 4, 0, 6, 4575, 5, 20, 8162, 27, 0, 4, 8188, 11, 0, 0, 8160, 0, 2, 7768, 8151, 24, 8188, 8165, 6316, 2, 7885, 8173, 0, 0, 8148, 8184, 0, 2, 8167, 4, 0, 5, 8172, 8187, 1, 0, 47, 0, 7] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16346, 12678, 16384, 16384] dup_rows_per_dst=[16884, 24744, 16576, 25124, 18728, 6, 16300, 16454] combine_rows_per_src=[16384, 16384, 16384, 16384, 16346, 12678, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=105468 dedup_ratio=0.500053 router_tokens_per_expert=[8157, 8192, 0, 0, 0, 12, 0, 0, 2, 1, 0, 0, 0, 0, 8187, 0, 8133, 8192, 8192, 0, 0, 0, 613, 0, 0, 8192, 15, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 4, 8191, 8192, 0, 7758, 0, 8043, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 8158, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16376, 16384, 16384, 16378, 16384, 16384] dup_rows_per_dst=[25618, 16380, 8726, 16282, 16390, 15522, 16086, 16082] combine_rows_per_src=[16384, 16384, 16376, 16384, 16384, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131064 payload_rows_remote=114680 dup_rows=131080 dup_rows_remote=114722 dedup_ratio=0.500031 router_tokens_per_expert=[1, 0, 1, 83, 1, 0, 1, 0, 6294, 0, 1, 0, 0, 752, 0, 0, 3, 0, 1, 29, 1, 1, 0, 1, 5186, 1, 8, 8184, 6, 0, 1, 8191, 6, 0, 0, 8185, 0, 0, 7578, 8072, 4, 8191, 8184, 4897, 1, 8096, 8186, 0, 0, 8177, 8189, 0, 2, 8174, 1, 0, 1, 8183, 8191, 0, 0, 5, 0, 2] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16378, 16384, 16384] dup_rows_per_dst=[16516, 14080, 16442, 10388, 14930, 25986, 16358, 16380] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16378, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131042 payload_rows_remote=114688 dup_rows=131102 dup_rows_remote=116674 dedup_ratio=0.500114 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4946, 0, 0, 0, 0, 1225, 0, 33, 0, 0, 0, 0, 0, 0, 2, 0, 6184, 0, 1, 8192, 1, 0, 0, 8191, 1, 0, 0, 8192, 0, 0, 7331, 8059, 0, 8192, 8191, 4980, 0, 8185, 8192, 0, 3, 8188, 8192, 4, 0, 8190, 0, 1, 0, 8191, 8192, 0, 2, 11, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16354, 16384, 16384, 16384] dup_rows_per_dst=[16380, 12408, 16388, 12372, 14428, 26328, 16390, 16408] combine_rows_per_src=[16384, 16384, 16384, 16384, 16354, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 8, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=917045248 dispatch_meta_bytes=916552 combine_payload_bytes=917045248 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=917961800 ingress_bytes=1269366784 link_bytes=1269366784 token_bytes=1835007048 weight_movement_bytes=352321536 total_nvlink_bytes=2187328584 weight_to_token_ratio=0.192000 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.107924131e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[4, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 4, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131058 payload_rows_remote=114674 dup_rows=131086 dup_rows_remote=115310 dedup_ratio=0.500053 router_tokens_per_expert=[124, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8183, 0, 0, 8192, 0, 8192, 12, 0, 4, 0, 8190, 0, 260, 6, 0, 2, 0, 8098, 0, 7974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8192, 8192, 0, 8187, 8192, 0, 0, 8157, 0, 0, 0, 8185, 0, 8149, 8181, 6, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16370, 16384] dup_rows_per_dst=[16632, 16366, 16932, 15776, 16374, 16404, 16314, 16288] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16370, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=908853248 dispatch_meta_bytes=884064 combine_payload_bytes=908853248 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=909737312 ingress_bytes=1261174784 link_bytes=1261174784 token_bytes=1818590560 weight_movement_bytes=352321536 total_nvlink_bytes=2170912096 weight_to_token_ratio=0.193733 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.062052350e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114678 dedup_ratio=0.500023 router_tokens_per_expert=[49, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8184, 0, 0, 8192, 0, 8192, 4, 0, 3, 0, 8179, 0, 347, 0, 0, 1, 0, 8083, 0, 7993, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 8, 0, 0, 0, 0, 8192, 8192, 0, 8190, 8192, 0, 0, 8166, 0, 0, 0, 8189, 0, 8183, 8143, 3, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16378, 16384] dup_rows_per_dst=[16482, 16368, 13088, 19748, 16386, 16400, 16332, 16274] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16378, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=880568 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940289976 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879699384 weight_movement_bytes=176160768 total_nvlink_bytes=2055860152 weight_to_token_ratio=0.093718 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939458560 dispatch_meta_bytes=917608 combine_payload_bytes=939458560 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940376168 ingress_bytes=1467940864 link_bytes=1467940864 token_bytes=1879834728 weight_movement_bytes=528482304 total_nvlink_bytes=2408317032 weight_to_token_ratio=0.281132 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.219856090e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=128430 payload_rows_remote=112096 dup_rows=133714 dup_rows_remote=112668 dedup_ratio=0.510078 router_tokens_per_expert=[649, 0, 0, 0, 22, 8191, 8171, 0, 0, 2, 0, 8188, 2, 0, 8179, 1, 8191, 1, 12, 0, 1, 8189, 6, 1624, 0, 1, 0, 23, 6279, 1, 8140, 1, 1, 0, 1, 1, 1, 2, 1, 1, 0, 6, 0, 1, 2, 0, 8185, 8191, 1, 8189, 8190, 0, 1, 8185, 0, 0, 1, 8191, 0, 7881, 8165, 0, 1, 0] payload_rows_per_dst=[13796, 16384, 16382, 16384, 16382, 16384, 16384, 16334] dup_rows_per_dst=[12324, 17104, 5022, 28888, 16280, 16594, 16456, 21046] combine_rows_per_src=[13796, 16384, 16382, 16384, 16382, 16384, 16384, 16334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=925448 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940449544 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879973640 weight_movement_bytes=528482304 total_nvlink_bytes=2408455944 weight_to_token_ratio=0.281112 gemm_rows_defined=131968 gemm_rows_charged=135168 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.023674 gemm_active_groups=12 gemm_peak_group_rows=30208 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=3 export_slots=3 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 3, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939409408 dispatch_meta_bytes=919936 combine_payload_bytes=939409408 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940329344 ingress_bytes=1115570176 link_bytes=1115570176 token_bytes=1879738752 weight_movement_bytes=176160768 total_nvlink_bytes=2055899520 weight_to_token_ratio=0.093716 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.246727324e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=917440 combine_payload_bytes=939474944 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940392384 ingress_bytes=1115635712 link_bytes=1115635712 token_bytes=1879867328 weight_movement_bytes=176160768 total_nvlink_bytes=2056028096 weight_to_token_ratio=0.093709 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.022727 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247094298e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=918290432 dispatch_meta_bytes=899056 combine_payload_bytes=918290432 prefetch_bytes=176160768 reduce_bytes=264241152 egress_bytes=919189488 ingress_bytes=1358692352 link_bytes=1358692352 token_bytes=1837479920 weight_movement_bytes=440401920 total_nvlink_bytes=2277881840 weight_to_token_ratio=0.239677 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.020833 gemm_active_groups=15 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.608110025e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130658 payload_rows_remote=114292 dup_rows=131486 dup_rows_remote=106028 dedup_ratio=0.501579 router_tokens_per_expert=[821, 0, 0, 0, 268, 8174, 8054, 1, 8, 26, 0, 8182, 22, 0, 7914, 9, 8179, 2, 210, 1, 2, 8166, 102, 4250, 0, 10, 1, 96, 4093, 11, 7919, 10, 0, 0, 9, 17, 25, 0, 8, 12, 0, 7, 0, 11, 13, 2, 8073, 8182, 11, 8175, 8170, 0, 10, 8167, 9, 0, 10, 8180, 0, 7480, 7963, 1, 6, 0] payload_rows_per_dst=[16364, 16384, 16366, 16380, 16384, 16384, 16382, 16014] dup_rows_per_dst=[18272, 15938, 25458, 7900, 16448, 16192, 16372, 14906] combine_rows_per_src=[16364, 16384, 16366, 16380, 16384, 16384, 16382, 16014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130860 payload_rows_remote=114476 dup_rows=131284 dup_rows_remote=115068 dedup_ratio=0.500809 router_tokens_per_expert=[1345, 2, 0, 0, 122, 8186, 8114, 0, 3, 10, 0, 8189, 8, 0, 8086, 4, 8187, 0, 88, 0, 2, 8176, 40, 4118, 0, 4, 0, 48, 3364, 4, 8125, 4, 5, 0, 4, 6, 9, 0, 3, 4, 0, 0, 0, 4, 3, 0, 8153, 8188, 3, 8182, 8186, 0, 4, 8188, 2, 0, 4, 8188, 0, 7633, 8071, 0, 3, 0] payload_rows_per_dst=[16380, 16384, 16376, 16370, 16384, 16384, 16384, 16198] dup_rows_per_dst=[19158, 16216, 24846, 6728, 16414, 16312, 16386, 15224] combine_rows_per_src=[16380, 16384, 16376, 16370, 16384, 16384, 16384, 16198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=126292 payload_rows_remote=109908 dup_rows=135852 dup_rows_remote=119466 dedup_ratio=0.518234 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 2680, 0, 0, 0, 0, 1855, 0, 43, 0, 0, 0, 1, 0, 0, 2, 0, 6501, 0, 12, 8192, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 7735, 8127, 0, 8192, 8189, 5800, 0, 8188, 8192, 0, 8, 8187, 8192, 5, 0, 8190, 0, 1, 0, 8191, 8192, 0, 0, 12, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 11610, 16384, 16382] dup_rows_per_dst=[16386, 25410, 16390, 30606, 15462, 390, 15200, 16008] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 11610, 16384, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130986 payload_rows_remote=114602 dup_rows=131158 dup_rows_remote=114774 dedup_ratio=0.500328 router_tokens_per_expert=[599, 1, 0, 0, 40, 8191, 8174, 0, 1, 1, 0, 8191, 2, 0, 8169, 1, 8191, 0, 24, 0, 0, 8189, 10, 2238, 0, 1, 0, 12, 5918, 1, 7964, 1, 9, 0, 1, 1, 2, 0, 1, 1, 0, 0, 0, 1, 1, 0, 8182, 8191, 0, 8190, 8190, 0, 1, 8191, 1, 0, 1, 8191, 0, 7928, 8069, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16300] dup_rows_per_dst=[17626, 16346, 5186, 27146, 16406, 16366, 16384, 15698] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114682 dup_rows=131078 dup_rows_remote=114696 dedup_ratio=0.500023 router_tokens_per_expert=[154, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8187, 0, 0, 8192, 0, 8192, 5, 0, 1, 0, 8160, 0, 2390, 2, 0, 1, 0, 6245, 0, 7749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 8192, 0, 8191, 8192, 0, 0, 8176, 0, 0, 0, 8191, 0, 8155, 8115, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16382, 16384] dup_rows_per_dst=[16692, 16374, 21112, 11614, 16382, 16394, 16352, 16158] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=936280064 dispatch_meta_bytes=881280 combine_payload_bytes=936280064 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=937161344 ingress_bytes=1288601600 link_bytes=1288601600 token_bytes=1873441408 weight_movement_bytes=352321536 total_nvlink_bytes=2225762944 weight_to_token_ratio=0.188061 gemm_rows_defined=132736 gemm_rows_charged=135168 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017992 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.215631071e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114664 dedup_ratio=0.500038 router_tokens_per_expert=[3094, 8171, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 7660, 0, 0, 0, 0, 8181, 3613, 2082, 0, 6, 8192, 0, 0, 0, 0, 11, 0, 8187, 2, 8189, 1, 0, 0, 8192, 8190, 0, 9, 8192, 0, 0, 8165, 8192, 0, 8153, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16374, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[22540, 16380, 10326, 16418, 16380, 16348, 16312, 16378] combine_rows_per_src=[16374, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 6] export_slots_per_peer=[0, 0, 0, 0, 4, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 8, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131034 payload_rows_remote=114650 dup_rows=131110 dup_rows_remote=114824 dedup_ratio=0.500145 router_tokens_per_expert=[3914, 8169, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6532, 0, 0, 0, 0, 8188, 4723, 1294, 0, 3, 8192, 0, 0, 0, 0, 7, 0, 8189, 0, 8191, 0, 0, 0, 8192, 8192, 0, 2, 8192, 0, 0, 8141, 8192, 0, 8179, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16346, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[7828, 16384, 25090, 16404, 16378, 16286, 16358, 16382] combine_rows_per_src=[16346, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=937787392 dispatch_meta_bytes=918176 combine_payload_bytes=937787392 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=938705568 ingress_bytes=1113948160 link_bytes=1113948160 token_bytes=1876492960 weight_movement_bytes=176160768 total_nvlink_bytes=2052653728 weight_to_token_ratio=0.093878 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.237644711e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131008 payload_rows_remote=114624 dup_rows=131136 dup_rows_remote=114748 dedup_ratio=0.500244 router_tokens_per_expert=[2042, 8135, 0, 1, 0, 0, 33, 0, 1, 1, 1, 1, 1, 0, 8185, 0, 6433, 0, 1, 0, 0, 8186, 5138, 3322, 0, 0, 8191, 1, 1, 0, 0, 5, 10, 8186, 0, 8191, 146, 1, 3, 8191, 8191, 6, 0, 8104, 0, 1, 7594, 8190, 1, 8190, 0, 0, 1, 0, 8191, 1, 1, 1, 1, 0, 0, 0, 8191, 1] payload_rows_per_dst=[16338, 16384, 16384, 16384, 16384, 16366, 16384, 16384] dup_rows_per_dst=[4084, 16378, 25902, 16358, 11396, 24030, 16600, 16388] combine_rows_per_src=[16338, 16384, 16384, 16384, 16384, 16366, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=6 unused_prefetch_slots=26 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=2 slot_utilization=0.187500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 6, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=2 dispatch_payload_bytes=900366336 dispatch_meta_bytes=917496 combine_payload_bytes=900366336 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=901283832 ingress_bytes=1252687872 link_bytes=1252687872 token_bytes=1801650168 weight_movement_bytes=352321536 total_nvlink_bytes=2153971704 weight_to_token_ratio=0.195555 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.019886 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.014529186e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=938819584 dispatch_meta_bytes=917504 combine_payload_bytes=938819584 prefetch_bytes=264241152 reduce_bytes=352321536 egress_bytes=939737088 ingress_bytes=1555382272 link_bytes=1555382272 token_bytes=1878556672 weight_movement_bytes=616562688 total_nvlink_bytes=2495119360 weight_to_token_ratio=0.328211 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.709491475e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939474944 dispatch_meta_bytes=917512 combine_payload_bytes=939474944 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940392456 ingress_bytes=1291796480 link_bytes=1291796480 token_bytes=1879867400 weight_movement_bytes=352321536 total_nvlink_bytes=2232188936 weight_to_token_ratio=0.187418 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233521066e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=7 export_slots=7 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 4, 0, 3, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917368 combine_payload_bytes=939442176 prefetch_bytes=264241152 reduce_bytes=0 egress_bytes=940359544 ingress_bytes=1203683328 link_bytes=1203683328 token_bytes=1879801720 weight_movement_bytes=264241152 total_nvlink_bytes=2144042872 weight_to_token_ratio=0.140569 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.740124195e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939212800 dispatch_meta_bytes=917896 combine_payload_bytes=939212800 prefetch_bytes=176160768 reduce_bytes=616562688 egress_bytes=940130696 ingress_bytes=1731936256 link_bytes=1731936256 token_bytes=1879343496 weight_movement_bytes=792723456 total_nvlink_bytes=2672066952 weight_to_token_ratio=0.421809 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.698120088e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=938999808 dispatch_meta_bytes=917488 combine_payload_bytes=938999808 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939917296 ingress_bytes=1115160576 link_bytes=1115160576 token_bytes=1878917104 weight_movement_bytes=176160768 total_nvlink_bytes=2055077872 weight_to_token_ratio=0.093757 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.021780 gemm_active_groups=15 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244433735e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131014 payload_rows_remote=114642 dup_rows=131130 dup_rows_remote=120716 dedup_ratio=0.500221 router_tokens_per_expert=[2859, 8135, 81, 0, 0, 7, 128, 1, 13, 10, 12, 9, 10, 0, 8102, 12, 4145, 0, 10, 0, 0, 8182, 5292, 3946, 0, 1, 8181, 17, 0, 0, 0, 3, 78, 8173, 1, 8182, 1686, 13, 45, 8174, 8181, 382, 0, 7635, 1, 18, 6653, 8122, 9, 8156, 0, 7, 9, 0, 8170, 1, 9, 11, 10, 0, 0, 0, 8179, 11] payload_rows_per_dst=[16382, 16384, 16372, 16384, 16384, 16340, 16384, 16384] dup_rows_per_dst=[22404, 16314, 10414, 16264, 19956, 13038, 16320, 16420] combine_rows_per_src=[16382, 16384, 16372, 16384, 16384, 16340, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131046 payload_rows_remote=114662 dup_rows=131098 dup_rows_remote=114756 dedup_ratio=0.500099 router_tokens_per_expert=[3075, 8165, 15, 2, 0, 1, 31, 0, 4, 4, 5, 4, 4, 0, 8151, 3, 3796, 0, 4, 0, 0, 8188, 5644, 4038, 0, 0, 8188, 4, 8, 1, 0, 0, 49, 8182, 0, 8188, 851, 5, 20, 8185, 8188, 153, 2, 7819, 0, 7, 7326, 8173, 4, 8188, 0, 3, 4, 0, 8184, 0, 4, 4, 5, 2, 0, 0, 8187, 4] payload_rows_per_dst=[16382, 16384, 16382, 16384, 16384, 16362, 16384, 16384] dup_rows_per_dst=[22572, 16342, 10582, 16364, 18200, 14252, 16382, 16404] combine_rows_per_src=[16382, 16384, 16382, 16384, 16384, 16362, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131062 payload_rows_remote=114678 dup_rows=131082 dup_rows_remote=114584 dedup_ratio=0.500038 router_tokens_per_expert=[57, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8180, 0, 0, 8192, 0, 8192, 6, 0, 1, 0, 8163, 0, 1559, 2, 0, 1, 0, 7038, 0, 7798, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 13, 0, 0, 0, 0, 8192, 8192, 0, 8189, 8192, 0, 0, 8175, 0, 0, 0, 8189, 0, 8173, 8178, 3, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16378, 16380] dup_rows_per_dst=[16498, 16360, 19458, 13294, 16384, 16410, 16506, 16172] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16378, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131040 payload_rows_remote=114656 dup_rows=131104 dup_rows_remote=114722 dedup_ratio=0.500122 router_tokens_per_expert=[3623, 8174, 0, 1, 0, 1, 9, 0, 1, 0, 1, 1, 1, 0, 8186, 1, 5277, 0, 1, 0, 0, 8191, 4908, 2887, 0, 0, 8191, 1, 1, 0, 0, 0, 8, 8191, 0, 8191, 207, 1, 5, 8190, 8191, 35, 4, 8089, 0, 2, 7739, 8186, 1, 8190, 0, 1, 1, 0, 8190, 0, 1, 1, 0, 0, 0, 0, 8191, 1] payload_rows_per_dst=[16360, 16384, 16384, 16384, 16384, 16376, 16384, 16384] dup_rows_per_dst=[7256, 16380, 26144, 16374, 16820, 15362, 16382, 16386] combine_rows_per_src=[16360, 16384, 16384, 16384, 16384, 16376, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939147264 dispatch_meta_bytes=941432 combine_payload_bytes=939147264 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940088696 ingress_bytes=1467629568 link_bytes=1467629568 token_bytes=1879235960 weight_movement_bytes=528482304 total_nvlink_bytes=2407718264 weight_to_token_ratio=0.281222 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=23936 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.218112962e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114688 dedup_ratio=0.500000 router_tokens_per_expert=[4584, 8173, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 7056, 0, 0, 0, 0, 8188, 3158, 1650, 0, 1, 8192, 0, 0, 0, 0, 5, 0, 8191, 1, 8192, 0, 0, 0, 8192, 8191, 0, 0, 8192, 0, 0, 8151, 8192, 0, 8176, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[18620, 16382, 14250, 16396, 16384, 16302, 16354, 16384] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=114686 dedup_ratio=0.500000 router_tokens_per_expert=[8189, 0, 0, 7860, 0, 0, 6838, 0, 4181, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 3, 8190, 0, 0, 0, 8187, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 6, 2128, 8192, 0, 3559, 8192, 6, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[29390, 8362, 16384, 16386, 16384, 16384, 16396, 11386] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=8 unused_prefetch_slots=24 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=4 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 8, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130926 payload_rows_remote=114542 dup_rows=131218 dup_rows_remote=114848 dedup_ratio=0.500557 router_tokens_per_expert=[8192, 1, 0, 7965, 0, 0, 5834, 0, 4599, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1690, 8192, 0, 4489, 8192, 3, 0, 0] payload_rows_per_dst=[16238, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[11002, 25582, 16386, 16384, 16386, 16370, 16384, 12724] combine_rows_per_src=[16238, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=8 export_slots=8 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 2, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=125734 payload_rows_remote=109358 dup_rows=136410 dup_rows_remote=103540 dedup_ratio=0.520363 router_tokens_per_expert=[8191, 59, 1, 7740, 0, 9, 5472, 0, 5318, 1, 38, 8096, 2, 0, 0, 0, 8191, 0, 147, 1, 1, 8190, 0, 3, 0, 1, 1, 8190, 0, 8190, 0, 8191, 0, 1, 0, 8191, 0, 0, 0, 0, 1, 7996, 1, 0, 0, 668, 1, 1, 4, 1, 8167, 0, 0, 1, 8191, 0, 2104, 8185, 0, 3332, 8191, 0, 2, 1] payload_rows_per_dst=[11062, 16384, 16384, 16384, 16384, 16376, 16384, 16376] dup_rows_per_dst=[20, 26908, 16400, 16422, 16380, 10824, 16586, 32870] combine_rows_per_src=[11062, 16384, 16384, 16384, 16384, 16376, 16384, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=2 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939311104 dispatch_meta_bytes=917672 combine_payload_bytes=939311104 prefetch_bytes=352321536 reduce_bytes=0 egress_bytes=940228776 ingress_bytes=1291632640 link_bytes=1291632640 token_bytes=1879539880 weight_movement_bytes=352321536 total_nvlink_bytes=2231861416 weight_to_token_ratio=0.187451 gemm_rows_defined=132864 gemm_rows_charged=135168 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2540 gemm_tail_pad_fraction=0.017045 gemm_active_groups=20 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.232603630e-05 prefetch_bytes_per_gemm_flop=1.972853535e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=2 dispatch_payload_bytes=939442176 dispatch_meta_bytes=917048 combine_payload_bytes=939442176 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=940359224 ingress_bytes=1644085248 link_bytes=1644085248 token_bytes=1879801400 weight_movement_bytes=704643072 total_nvlink_bytes=2584444472 weight_to_token_ratio=0.374850 gemm_rows_defined=131840 gemm_rows_charged=135168 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.024621 gemm_active_groups=13 gemm_peak_group_rows=32128 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=9.206191114e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939261952 dispatch_meta_bytes=917512 combine_payload_bytes=939261952 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940179464 ingress_bytes=1115422720 link_bytes=1115422720 token_bytes=1879441416 weight_movement_bytes=176160768 total_nvlink_bytes=2055602184 weight_to_token_ratio=0.093730 gemm_rows_defined=132608 gemm_rows_charged=135168 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.018939 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.245901632e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=2 export_slots=2 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917504 combine_payload_bytes=939524096 prefetch_bytes=88080384 reduce_bytes=440401920 egress_bytes=940441600 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879965696 weight_movement_bytes=528482304 total_nvlink_bytes=2408448000 weight_to_token_ratio=0.281113 gemm_rows_defined=132480 gemm_rows_charged=135168 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.019886 gemm_active_groups=19 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=4.932133838e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=917496 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=940441592 ingress_bytes=1291845632 link_bytes=1291845632 token_bytes=1879965688 weight_movement_bytes=352321536 total_nvlink_bytes=2232287224 weight_to_token_ratio=0.187408 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=7.233796296e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938328064 dispatch_meta_bytes=917560 combine_payload_bytes=938328064 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939245624 ingress_bytes=1114488832 link_bytes=1114488832 token_bytes=1877573688 weight_movement_bytes=176160768 total_nvlink_bytes=2053734456 weight_to_token_ratio=0.093824 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32000 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.240672249e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=895860736 dispatch_meta_bytes=851592 combine_payload_bytes=895860736 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=896712328 ingress_bytes=1248182272 link_bytes=1248182272 token_bytes=1792573064 weight_movement_bytes=352321536 total_nvlink_bytes=2144894600 weight_to_token_ratio=0.196545 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.989299706e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=130900 payload_rows_remote=114516 dup_rows=131244 dup_rows_remote=113210 dedup_ratio=0.500656 router_tokens_per_expert=[8174, 325, 9, 6879, 0, 379, 4180, 0, 5881, 9, 159, 7508, 18, 0, 0, 0, 8179, 0, 822, 0, 8, 8174, 0, 26, 7, 2, 19, 8161, 0, 8132, 0, 8155, 0, 10, 7, 8181, 1, 71, 7, 0, 12, 7288, 12, 0, 0, 2108, 13, 9, 134, 13, 7987, 0, 0, 18, 8180, 0, 2028, 8081, 0, 3509, 8162, 4, 21, 10] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16212] dup_rows_per_dst=[23508, 10766, 18034, 16246, 16492, 18824, 16280, 11094] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29893149192 prefetch_bytes=3170893824 reduce_bytes=3523215360 weight_movement_bytes=6694109184 egress_bytes=14953873928 ingress_bytes=21633384448 link_bytes=21633384448 total_nvlink_bytes=36587258376 weight_to_token_ratio=0.223935 prefetch_share_of_ingress=0.146574 dedup_ratio=0.503012 prefetch_slots=72 unused_prefetch_slots=440 slot_utilization=0.140625 reduce_slots=40 remote_demand=36 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020123 gemm_inner_pad_upper_bound=36068 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.571120168e-05 prefetch_bytes_per_gemm_flop=1.109730114e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29879273144 prefetch_bytes=2906652672 reduce_bytes=4139778048 weight_movement_bytes=7046430720 egress_bytes=14946928312 ingress_bytes=21978775552 link_bytes=21978775552 total_nvlink_bytes=36925703864 weight_to_token_ratio=0.235830 prefetch_share_of_ingress=0.132248 dedup_ratio=0.503252 prefetch_slots=66 unused_prefetch_slots=446 slot_utilization=0.128906 reduce_slots=47 remote_demand=33 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020833 gemm_inner_pad_upper_bound=34798 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.691998044e-05 prefetch_bytes_per_gemm_flop=1.017252604e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29301777120 prefetch_bytes=2378170368 reduce_bytes=2466250752 weight_movement_bytes=4844421120 egress_bytes=14658003680 ingress_bytes=19488194560 link_bytes=19488194560 total_nvlink_bytes=34146198240 weight_to_token_ratio=0.165329 prefetch_share_of_ingress=0.122031 dedup_ratio=0.511496 prefetch_slots=54 unused_prefetch_slots=458 slot_utilization=0.105469 reduce_slots=28 remote_demand=27 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020656 gemm_inner_pad_upper_bound=34798 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.820359673e-05 prefetch_bytes_per_gemm_flop=8.322975852e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=2 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 2, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131008 payload_rows_remote=114624 dup_rows=131136 dup_rows_remote=121316 dedup_ratio=0.500244 router_tokens_per_expert=[8184, 190, 4, 6770, 3, 255, 4329, 0, 5150, 4, 95, 7845, 7, 0, 1, 0, 8187, 0, 471, 0, 4, 8185, 0, 9, 1, 1, 8, 8182, 0, 8180, 0, 8185, 0, 4, 3, 8188, 0, 17, 1, 0, 5, 7885, 4, 0, 0, 1788, 5, 4, 36, 5, 8093, 0, 0, 7, 8188, 0, 2055, 8150, 0, 4202, 8170, 0, 8, 4] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16322] dup_rows_per_dst=[23086, 9820, 17328, 16368, 16406, 19338, 16274, 12516] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 16384, 16384, 16322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131066 payload_rows_remote=114684 dup_rows=131078 dup_rows_remote=105186 dedup_ratio=0.500023 router_tokens_per_expert=[4785, 8160, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 7726, 0, 0, 0, 0, 8185, 2985, 1232, 0, 3, 8192, 0, 0, 0, 0, 8, 0, 8189, 0, 8192, 1, 0, 0, 8192, 8191, 0, 1, 8192, 0, 0, 7881, 8192, 0, 8181, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[25892, 16512, 8378, 16494, 15558, 15550, 16364, 16330] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16380, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938115072 dispatch_meta_bytes=910904 combine_payload_bytes=938115072 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=939025976 ingress_bytes=1466597376 link_bytes=1466597376 token_bytes=1877141048 weight_movement_bytes=528482304 total_nvlink_bytes=2405623352 weight_to_token_ratio=0.281536 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.020833 gemm_active_groups=14 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.212333118e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=127766 payload_rows_remote=111382 dup_rows=134378 dup_rows_remote=118000 dedup_ratio=0.512611 router_tokens_per_expert=[8191, 59, 1, 7595, 1, 90, 6442, 0, 3545, 1, 16, 8104, 2, 0, 0, 0, 8191, 0, 155, 0, 1, 8190, 0, 2, 0, 0, 1, 8190, 0, 8188, 0, 8190, 0, 1, 1, 8191, 0, 3, 1, 0, 1, 7739, 1, 0, 0, 628, 1, 1, 12, 1, 8176, 0, 0, 1, 8191, 0, 2000, 8180, 0, 4597, 8188, 0, 2, 1] payload_rows_per_dst=[13098, 16384, 16384, 16382, 16384, 16384, 16384, 16366] dup_rows_per_dst=[1912, 23334, 16694, 16376, 16390, 16734, 16378, 26560] combine_rows_per_src=[13098, 16384, 16384, 16382, 16384, 16384, 16384, 16366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131068 payload_rows_remote=114684 dup_rows=131076 dup_rows_remote=114696 dedup_ratio=0.500015 router_tokens_per_expert=[8191, 0, 0, 7798, 0, 0, 5053, 0, 5597, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8191, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 2, 2312, 8192, 0, 3850, 8192, 6, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[23534, 13364, 16384, 16382, 16380, 16308, 16388, 12336] combine_rows_per_src=[16380, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=938999808 dispatch_meta_bytes=943760 combine_payload_bytes=938999808 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=939943568 ingress_bytes=1115160576 link_bytes=1115160576 token_bytes=1878943376 weight_movement_bytes=176160768 total_nvlink_bytes=2055104144 weight_to_token_ratio=0.093755 gemm_rows_defined=132096 gemm_rows_charged=135168 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.022727 gemm_active_groups=16 gemm_peak_group_rows=32512 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.244433735e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=879480 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940370808 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879862136 weight_movement_bytes=176160768 total_nvlink_bytes=2056022904 weight_to_token_ratio=0.093709 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.020833 gemm_active_groups=17 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29838108336 prefetch_bytes=2554331136 reduce_bytes=4139778048 weight_movement_bytes=6694109184 egress_bytes=14926374576 ingress_bytes=21605842944 link_bytes=21605842944 total_nvlink_bytes=36532217520 weight_to_token_ratio=0.224348 prefetch_share_of_ingress=0.118224 dedup_ratio=0.503899 prefetch_slots=58 unused_prefetch_slots=454 slot_utilization=0.113281 reduce_slots=47 remote_demand=29 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020597 gemm_inner_pad_upper_bound=34290 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.561481360e-05 prefetch_bytes_per_gemm_flop=8.939492582e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=912441344 dispatch_meta_bytes=917528 combine_payload_bytes=912441344 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=913358872 ingress_bytes=1088602112 link_bytes=1088602112 token_bytes=1825800216 weight_movement_bytes=176160768 total_nvlink_bytes=2001960984 weight_to_token_ratio=0.096484 gemm_rows_defined=132224 gemm_rows_charged=135168 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.021780 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.095717422e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939491328 dispatch_meta_bytes=917520 combine_payload_bytes=939491328 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=940408848 ingress_bytes=1115652096 link_bytes=1115652096 token_bytes=1879900176 weight_movement_bytes=176160768 total_nvlink_bytes=2056060944 weight_to_token_ratio=0.093708 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.020833 gemm_active_groups=16 gemm_peak_group_rows=32768 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=6.247186041e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29822840448 prefetch_bytes=3258974208 reduce_bytes=2290089984 weight_movement_bytes=5549064192 egress_bytes=14918725248 ingress_bytes=20453179392 link_bytes=20453179392 total_nvlink_bytes=35371904640 weight_to_token_ratio=0.186068 prefetch_share_of_ingress=0.159338 dedup_ratio=0.503954 prefetch_slots=74 unused_prefetch_slots=438 slot_utilization=0.144531 reduce_slots=26 remote_demand=37 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020301 gemm_inner_pad_upper_bound=35941 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.158079189e-05 prefetch_bytes_per_gemm_flop=1.140555950e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29675008416 prefetch_bytes=2906652672 reduce_bytes=1761607680 weight_movement_bytes=4668260352 egress_bytes=14844785056 ingress_bytes=19498483712 link_bytes=19498483712 total_nvlink_bytes=34343268768 weight_to_token_ratio=0.157313 prefetch_share_of_ingress=0.149071 dedup_ratio=0.505992 prefetch_slots=66 unused_prefetch_slots=446 slot_utilization=0.128906 reduce_slots=20 remote_demand=33 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020893 gemm_inner_pad_upper_bound=34544 gemm_flops_step=285735584268288 bytes_per_gemm_flop=6.823960607e-05 prefetch_bytes_per_gemm_flop=1.017252604e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29913437104 prefetch_bytes=3347054592 reduce_bytes=3082813440 weight_movement_bytes=6429868032 egress_bytes=14964052912 ingress_bytes=21379252224 link_bytes=21379252224 total_nvlink_bytes=36343305136 weight_to_token_ratio=0.214949 prefetch_share_of_ingress=0.156556 dedup_ratio=0.502957 prefetch_slots=76 unused_prefetch_slots=436 slot_utilization=0.148438 reduce_slots=35 remote_demand=38 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020478 gemm_inner_pad_upper_bound=35433 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.482180520e-05 prefetch_bytes_per_gemm_flop=1.171381787e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 E=64 B=8 K=16 R=8 NvS=67568 hidden_size=4096 assignments=131072 payload_rows=131072 payload_rows_remote=114688 dup_rows=131072 dup_rows_remote=103694 dedup_ratio=0.500000 router_tokens_per_expert=[8192, 0, 0, 7959, 0, 0, 4269, 0, 6467, 0, 0, 8192, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8191, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 8120, 0, 0, 0, 29, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 1, 1240, 8192, 0, 4669, 8192, 14, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[27378, 12936, 13462, 16512, 16254, 16298, 16386, 11846] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 prefetch_calls=4 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=2 reduce_slots=4 export_slots=4 remote_demand=2 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=44040192 grad_bytes_per_expert=88080384 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=2 dispatch_payload_bytes=939524096 dispatch_meta_bytes=873528 combine_payload_bytes=939524096 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=940397624 ingress_bytes=1468006400 link_bytes=1468006400 token_bytes=1879921720 weight_movement_bytes=528482304 total_nvlink_bytes=2408404024 weight_to_token_ratio=0.281119 gemm_rows_defined=132352 gemm_rows_charged=135168 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.020833 gemm_active_groups=18 gemm_peak_group_rows=23296 gemm_flops_fwd=5952824672256 gemm_flops_step=17858474016768 bytes_per_gemm_flop=8.220223064e-05 prefetch_bytes_per_gemm_flop=9.864267677e-06 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=16 plans=32 dispatch_calls=64 combine_calls=64 prefetch_calls=64 reduce_calls=32 token_bytes=29439975240 prefetch_bytes=3170893824 reduce_bytes=2290089984 weight_movement_bytes=5460983808 egress_bytes=14727176008 ingress_bytes=20173783040 link_bytes=20173783040 total_nvlink_bytes=34900959048 weight_to_token_ratio=0.185496 prefetch_share_of_ingress=0.157179 dedup_ratio=0.509506 prefetch_slots=72 unused_prefetch_slots=440 slot_utilization=0.140625 reduce_slots=26 remote_demand=36 gemm_rows_charged=2162688 gemm_tail_pad_fraction=0.020656 gemm_inner_pad_upper_bound=34798 gemm_flops_step=285735584268288 bytes_per_gemm_flop=7.060297755e-05 prefetch_bytes_per_gemm_flop=1.109730114e-05 anomalies=none - [2026-09-16 21:39:35] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 612.1 | throughput per GPU (TFLOP/s/GPU): 544.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.873632E+00 | loss scale: 1.0 | grad norm: 5.303 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-09-16 21:39:35 -[rank0]:[W916 21:39:36.141531027 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W916 21:39:38.400595033 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W916 21:39:38.423780069 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W916 21:39:38.486142782 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W916 21:39:38.876073793 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W916 21:39:38.964545260 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W916 21:39:38.964916270 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W916 21:39:39.295795821 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0916 21:39:41.293000 40186 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0916 21:39:41.293000 40186 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40370 closing signal SIGTERM -W0916 21:39:41.293000 40186 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40371 closing signal SIGTERM -W0916 21:39:41.294000 40186 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40372 closing signal SIGTERM -W0916 21:39:41.295000 40186 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40373 closing signal SIGTERM -W0916 21:39:41.295000 40186 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40374 closing signal SIGTERM -W0916 21:39:41.296000 40186 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40375 closing signal SIGTERM -W0916 21:39:41.296000 40186 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40376 closing signal SIGTERM -W0916 21:39:41.297000 40186 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40377 closing signal SIGTERM -Traceback (most recent call last): - File "/usr/local/bin/torchrun", line 8, in - sys.exit(main()) - ^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper - return f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main - run(args) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run - elastic_launch( - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent - result = agent.run() - ^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper - result = f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run - result = self._invoke_run(role) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run - time.sleep(monitor_interval) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 40186 got signal: 15 +[21:15:57] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[21:15:57] WARN: MoonEP source is on 'megatron-integration', need 'megatron-ddp-symmetric-params' +[21:15:57] ERROR: MoonEP source must be /workspace/MoonEP @ megatron-ddp-symmetric-params diff --git a/granularity_next_b200/tiera_moonep_probe_refresh/study.log b/granularity_next_b200/tiera_moonep_probe_refresh/study.log index 04ebe10ba2e18709fc71155f2e2abdb0f2c5af11..be23c644ad325472e849b2aa975eda9606efecb7 100644 --- a/granularity_next_b200/tiera_moonep_probe_refresh/study.log +++ b/granularity_next_b200/tiera_moonep_probe_refresh/study.log @@ -174,4 +174,34 @@ moonep/tierbprobe/tierbprobe_gbs16/G4__moonep__match forced none 1.0177 ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional. The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally. no memory-dump runs found under /workspace/granularity_next_b200/tiera_moonep_probe_refresh +Already logged in to Hugging Face as: AbhishekGhosh1998 + uploaded: /workspace/granularity_next_b200/tiera_moonep_probe_refresh -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tiera_moonep_probe_refresh +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main + +=== STUDY COMPLETE + results : /workspace/granularity_next_b200/tiera_moonep_probe_refresh + status : /workspace/granularity_next_b200/tiera_moonep_probe_refresh/STATUS.txt + steps : /workspace/granularity_next_b200/tiera_moonep_probe_refresh/steps.tsv + hub : https://huggingface.co/datasets/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8 + +status campaign phase gbs started ended result_dir +OK moonep tierbprobe 16 2026-09-16T21:33:28+00:00 2026-09-16T21:43:29+00:00 /workspace/granularity_next_b200/tiera_moonep_probe_refresh/moonep/tierbprobe +study_root: /workspace/granularity_next_b200/tiera_moonep_probe_refresh +gbs: 16 +stages: 7 +nsys: 1 +hf_repo: MoE-CUDA-Graph-Analysis/tierb-b200-gbs8 +hf_remote_root: granularity_next_b200/tiera_moonep_probe_refresh +started: 2026-09-21T05:22:00+00:00 +driver_repo: /workspace/blackwell_run_scripts @ b8bd100 + +=== STAGE 7 -- Tier B probe and zero-drop capacity selection + +=== ROLL-UP +no successful runs to compare +8 runs ok=0 oom=0 -> /workspace/granularity_next_b200/tiera_moonep_probe_refresh/collected_all.tsv + +no probe telemetry found under /workspace/granularity_next_b200/tiera_moonep_probe_refresh + expected instrumented branches: token-drop-analysis / ultraep-token-drop-analysis / moonep_backend_token_routing_analysis +no memory-dump runs found under /workspace/granularity_next_b200/tiera_moonep_probe_refresh Already logged in to Hugging Face as: AbhishekGhosh1998